大数据实验室

# 下载scala基础镜像
docker pull gudaoxuri/scala_2.10
# 下载mysql镜像
docker pull mysql
# 下载Mongo镜像
docker pull mongo

# 从本地添加scala基础镜像
docker load -i <your path>/images/scala_2.10_image.tar
# 从本地添加mysql镜像
docker load -i <your path>/images/mysql_image.tar
# 从本地添加mongo镜像
docker load -i <your path>/images/mongo_image.tar

# 创建mysql容器，开放3306端口，用户名root，密码123456
docker run --name mysql -e MYSQL_ROOT_PASSWORD=123456 -p 3306:3306 -d mysql
# 创建mongo容器，开放27017端口，无需认证
docker run --name mongo -p 27017:27017 -d mongo
# 创建scala基础容器，开放大数据服务需要的几个重要端口
docker run --name bd-lab -h bd -d \
     -p 2181:2181 \
     -p 6066:6066 \
     -p 7077:7077 \
     -p 8088:8088 \
     -p 8041:8041 \
     -p 19888:19888 \
     -p 60010:60010 \
     -p 60030:60030 \
     -p 4040:4040 \
     -p 8080:8080 \
     -p 8081:8081 \
     -p 9000:9000 \
     -p 7203:7203 \
     -p 9092:9092 \
     -p 50010:50010 \
     -p 50070:50070 \
     -p 50075:50075 \
     -p 10000:10000 \
     -p 28081:28081 \
     -p 222:22 \
     -v /c/Users/i/docker_data/bd-lab:/opt/res \
     gudaoxuri/scala_2.10

ONBOOT=yes
BOOTPROTO=static
IPADDR=<ip>
NETMASK=<mask>
GATEWAY=<getway>

ifup eth0

nameserver 180.76.76.76
nameserver 8.8.8.8

ifconfig
ping www.baidu.com

# 测试环境，可以关闭防火墙
chkconfig iptables off
service iptables stop

yum install -y wget

# 安装java
wget -P /opt/env/ --no-check-certificate --no-cookies --header "Cookie: oraclelicense=accept-securebackup-cookie"  http://download.oracle.com/otn-pub/java/jdk/8u60-b27/jdk-8u60-linux-x64.tar.gz
tar -xzf /opt/env/jdk-8u60-linux-x64.tar.gz -C /opt/env/
rm -rf /opt/env/jdk-8u60-linux-x64.tar.gz
mv /opt/env/jdk1.8.0_60 /opt/env/java
echo "export JAVA_HOME=/opt/env/java" >> /etc/profile

# 安装Scala
wget -P /opt/env/ http://downloads.typesafe.com/scala/2.10.6/scala-2.10.6.tgz
tar -xzf /opt/env/scala-2.10.6.tgz -C /opt/env/
rm -rf /opt/env/scala-2.10.6.tgz
mv /opt/env/scala-2.10.6 /opt/env/scala
echo "export SCALA_HOME=/opt/env/scala" >> /etc/profile

echo 'export PATH=$PATH:$JAVA_HOME/bin:$SCALA_HOME/bin' >> /etc/profile
source /etc/profile

# 安装MySQL
yum install -y mysql-server mysql mysql-devel
service mysqld start
# mysqladmin -u root password 123456
mysql -u root -p
grant all on *.* to'root'@'%' identified by '123456';
flush privileges;

# 安装Mongo
vi /etc/yum.repos.d/mongodb-org-3.2.repo
  [mongodb-org-3.2]
  name=MongoDB Repository
  baseurl=https://repo.mongodb.org/yum/redhat/$releasever/mongodb-org/3.2/x86_64/
  gpgcheck=1
  enabled=1
  gpgkey=https://www.mongodb.org/static/pgp/server-3.2.asc

yum install -y mongodb-org

vi /etc/mongod.conf
  bind_ip = 0.0.0.0

service mongod start

vi /etc/selinux/config
   SELLINUX=disabled

yum install -y samba
vi /etc/samba/smb.conf
    # 开启匿名访问
    security = share
    # 修改访问目录
    [public]
    comment = Public Stuff
    path = /share
    availabe = yes
    browseable = yes
    public = yes
    writable = yes
    printable = no
    write list = +staff

mkdir /share
chmod 777 /share
/etc/init.d/smb start

# 创建用户user,密码123456
adduser user
passwd user
mkdir -p /opt/workspaces/
chown user: /opt/workspaces/

# 进入user账号
su - user

# 切换到工作空间
cd /opt/workspaces
# 创建各框架lib目录
mkdir frameworks
# 创建各框架数据目录
mkdir data
# 创建各框架日志目录
mkdir logs
# 创建应用包目录
mkdir apps

# 创建SSH免密钥登录
ssh-keygen -t dsa -P '' -f ~/.ssh/id_dsa
cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys
chmod 0600 ~/.ssh/authorized_keys

# 切换到工作空间
cd /opt/workspaces
mkdir data/hadoop
# 创建Hadoop NameNode目录
mkdir -p data/hadoop/hdfs/nn
# 创建Hadoop DataNode目录
mkdir -p data/hadoop/hdfs/dn
# 创建Hadoop临时目录
mkdir data/hadoop/tmp
# 创建Hadoop日志目录
mkdir logs/hadoop

wget http://mirrors.hust.edu.cn/apache/hadoop/common/hadoop-2.7.2/hadoop-2.7.2.tar.gz
tar -zxf hadoop-2.7.2.tar.gz
rm -rf hadoop-2.7.2.tar.gz
mv hadoop-2.7.2 ./frameworks/hadoop

# 添加JDK目录
export JAVA_HOME=/opt/env/java
# 指定Hadoop日志写入到先前定义的目录
export HADOOP_LOG_DIR=/opt/workspaces/logs/hadoop

<!--指定HDFS的地址，bd为对应的hostname，请自行修改-->
<property>
    <name>fs.defaultFS</name>
    <value>hdfs://bd:9000</value>
</property>
<!--指定Hadoop的临时目录到先前定义的目录-->
<property>
    <name>hadoop.tmp.dir</name>
    <value>file:/opt/workspaces/data/hadoop/tmp</value>
</property>

<!--副本数，默认是3，由于只有一个节点，所以不需要备份-->
<property>
    <name>dfs.replication</name>
    <value>1</value>
</property>
<!--指定NameNode目录到先前定义的目录-->
<property>
    <name>dfs.namenode.name.dir</name>
    <value>file:/opt/workspaces/data/hadoop/hdfs/nn</value>
</property>
<!--指定DataNode目录到先前定义的目录-->
<property>
    <name>dfs.datanode.data.dir</name>
    <value>file:/opt/workspaces/data/hadoop/hdfs/dn</value>
</property>
<!--实验环境，禁用权限认证-->
<property>
    <name>dfs.permissions.enabled</name>
    <value>false</value>
</property>

./frameworks/hadoop/bin/hdfs namenode -format

# 启动NameNode
./frameworks/hadoop/sbin/hadoop-daemon.sh start namenode
# 启动DataNode
./frameworks/hadoop/sbin/hadoop-daemon.sh start datanode
# 停止NameNode
./frameworks/hadoop/sbin/hadoop-daemon.sh stop namenode
# 停止DataNode
./frameworks/hadoop/sbin/hadoop-daemon.sh stop datanode

# 查看HDFS文件
./frameworks/hadoop/bin/hadoop fs -ls /

# 上传文件到HDFS
hadoop fs -put localfile /user/hadoop/hadoopfile hdfs://<host>:<port>/<path>
hadoop fs -put localfile1 localfile2 /user/hadoop/hadoopdir
hadoop fs -put localfile hdfs://nn.example.com/hadoop/hadoopfile

# 创建HDFS目录
hadoop fs -mkdir /user/hadoop/dir1 /user/hadoop/dir2
hadoop fs -mkdir hdfs://nn1.example.com/user/hadoop/dir hdfs://nn2.example.com/user/hadoop/dir

# 查看HDFS目录
hadoop fs -ls /user/hadoop/file1

# 查看HDFS文件内容
hadoop fs -cat hdfs://nn1.example.com/file1 hdfs://nn2.example.com/file2
hadoop fs -cat file:///file3 /user/hadoop/file4

# 修改HDFS文件所有者
hadoop fs -chown [-R] [OWNER][:[GROUP]] URI [URI ]

# 修改HDFS文件权限
hadoop fs -chmod [-R] <MODE[,MODE]... | OCTALMODE> URI [URI ...]

# 获取HDFS到本地
hadoop fs -copyToLocal [-ignorecrc] [-crc] URI <localdst>

# 切换到工作空间
cd /opt/workspaces
# 创建Spark数据目录
mkdir data/spark
# 创建Spark日志目录
mkdir logs/spark

wget http://mirrors.hust.edu.cn/apache/spark/spark-1.6.1/spark-1.6.1-bin-hadoop2.6.tgz
tar -zxf spark-1.6.1-bin-hadoop2.6.tgz
rm -rf spark-1.6.1-bin-hadoop2.6.tgz
mv spark-1.6.1-bin-hadoop2.6 ./frameworks/spark

export SPARK_MASTER_IP=bd
export SPARK_MASTER_PORT=7077
export MASTER=spark://${SPARK_MASTER_IP}:${SPARK_MASTER_PORT}
# 指定Spark数据目录
export SPARK_LOCAL_DIRS=/opt/workspaces/data/spark/
# 指定Spark日志目录
export SPARK_LOG_DIR=/opt/workspaces/logs/spark/
# 指定JDK目录
export JAVA_HOME=/opt/env/java
# 指定Scala目录
export SCALA_HOME=/opt/env/scala

./frameworks/spark/sbin/start-all.sh

# 执行圆周率计算示例
./frameworks/spark/bin/run-example  org.apache.spark.examples.SparkPi

./frameworks/spark/bin/spark-submit \
  --class org.apache.spark.examples.SparkPi \
  --master spark://bd:6066\
  --deploy-mode cluster \
  --driver-memory 512M \
  --executor-memory 256M \  #如果运行出错请改大些
  ./frameworks/spark/lib/spark-examples-1.6.1-hadoop2.6.0.jar \
  1000

val conf = new SparkConf()
             // 指定使用2个本地线程来运行，本地模式下，我们可以使用n个线程（n >= 1），但在像Spark Streaming这样的场景下，我们可能需要多个线程
             .setMaster("local[2]")
             .setAppName("CountingSheep")
val sc = new SparkContext(conf)

./bin/spark-submit --name "My app" --master local[4] --conf spark.eventLog.enabled=false
  --conf "spark.executor.extraJavaOptions=-XX:+PrintGCDetails -XX:+PrintGCTimeStamps" myApp.jar

val conf = new SparkConf()
             .setMaster(...)
             .setAppName(...)
             .set("spark.cores.max", "10")
val sc = new SparkContext(conf)

export SPARK_MASTER_OPTS="-Dspark.deploy.defaultCores=<value>"

http://spark-internals.books.yourtion.com/index.html +
此文写得挺详细，由于撰写时间较早，与新版本有出入（比如Shuffle Manager现在默认是Sort-based），但不影响对Spark的整体理解。

# 切换到工作空间
cd /opt/workspaces
# 创建Zookeeper数据目录
mkdir data/zookeeper
# 创建Zookeeper日志目录
mkdir logs/zookeeper

wget http://mirrors.hust.edu.cn/apache/zookeeper/current/zookeeper-3.4.8.tar.gz
tar -zxf zookeeper-3.4.8.tar.gz
rm -rf zookeeper-3.4.8.tar.gz
mv zookeeper-3.4.8 ./frameworks/zookeeper

cat>./frameworks/zookeeper/conf/zoo.cfg<<EOF
# 服务器之间或客户端与服务器之间维持心跳的时间间隔
tickTime=2000
# 数据目录
dataDir=/opt/workspaces/data/zookeeper
# 日志目录
dataLogDir=/opt/workspaces/logs/zookeeper
# 客户端连接端口
clientPort=2181
EOF

./frameworks/zookeeper/bin/zkServer.sh start

./frameworks/zookeeper/bin/zkCli.sh -server localhost:2181
   ls /
   [zookeeper]

# 创建文件并初始内容
create /zk "test"

# 获取文件内容
get /zk

# 修改文件内容
set /zk "new"

# 显示目录
ls /

# 删除文件
delete /zk

# 退出客户端
quit

# 切换到工作空间
cd /opt/workspaces
# 创建HBase日志目录
mkdir logs/hbase

wget http://mirrors.cnnic.cn/apache/hbase/1.2.1/hbase-1.2.1-bin.tar.gz
tar -zxf hbase-1.2.1-bin.tar.gz
rm -rf hbase-1.2.1-bin.tar.gz
mv hbase-1.2.1 ./frameworks/hbase

export JAVA_HOME=/opt/env/java
export HBASE_LOG_DIR=/opt/workspaces/logs/hbase
export HBASE_MANAGES_ZK=false

<property>
  <name>hbase.rootdir</name>
  <value>hdfs://bd:9000/hbase</value>
</property>
<property>
  <name>hbase.cluster.distributed</name>
  <value>true</value>
</property>
<property>
  <name>hbase.zookeeper.property.clientPort</name>
  <value>2181</value>
</property>
<property>
  <name>hbase.zookeeper.quorum</name>
  <value>bd</value>
</property>
<property>
  <name>hbase.zookeeper.property.dataDir</name>
  <value>/opt/workspaces/data/zookeeper</value>
</property>
<property>
  <name>dfs.support.append</name>
  <value>true</value>
</property>

./frameworks/hbase/bin/start-hbase.sh

./frameworks/hbase/bin/hbase shell
create 'test', 'cf'
list 'test'
put 'test', 'row1', 'cf:a', 'value1'
put 'test', 'row2', 'cf:b', 'value2'
put 'test', 'row3', 'cf:c', 'value3'
scan 'test'
get 'test', 'row1'
disable 'test'
enable 'test'
drop 'test'

# 切换到工作空间
cd /opt/workspaces
# 创建Hive数据目录
mkdir data/hive
# 创建Hive日志目录
mkdir logs/hive

wget http://mirrors.cnnic.cn/apache/hive/hive-1.2.1/apache-hive-1.2.1-bin.tar.gz
tar -zxf apache-hive-1.2.1-bin.tar.gz
rm -rf apache-hive-1.2.1-bin.tar.gz
mv apache-hive-1.2.1-bin ./frameworks/hive

export HIVE_HOME=/opt/workspaces/frameworks/hive
export PATH=$PATH:$HIVE_HOME/bin

mysql -u root -p
create database hive;
grant all on *.* to'hive'@'%' identified by 'hive';  #账号hive,密码hive,所有IP都可以访问（%）
flush privileges;
exit;

./frameworks/hadoop/bin/hadoop fs -mkdir /tmp
./frameworks/hadoop/bin/hadoop fs -mkdir -p /user/hive/warehouse
./frameworks/hadoop/bin/hadoop fs -chmod 777 /tmp
./frameworks/hadoop/bin/hadoop fs -chmod 777 /user/hive/warehouse

cat>./frameworks/hive/conf/hive-env.sh<<EOF
export HADOOP_HEAPSIZE=1024
HADOOP_HOME=/opt/workspaces/frameworks/hadoop
export HIVE_CONF_DIR=/opt/workspaces/frameworks/hive/conf
export HIVE_AUX_JARS_PATH=/opt/workspaces/frameworks/hive/lib
EOF
cat>./frameworks/hive/conf/hive-site.xml<<EOF
<configuration>
    <property>
        <name>javax.jdo.option.ConnectionURL</name>
        <value>jdbc:mysql://`<Docker Host IP>`:3306/hive</value>
    </property>
    <property>
        <name>javax.jdo.option.ConnectionDriverName</name>
        <value>com.mysql.jdbc.Driver</value>
    </property>
    <property>
        <name>javax.jdo.option.ConnectionUserName</name>
        <value>hive</value>
    </property>
    <property>
        <name>javax.jdo.option.ConnectionPassword</name>
        <value>hive</value>
    </property>
    <property>
        <name>hive.metastore.uris</name>
        <value>thrift://bd:9083</value>
    </property>
    <property>
        <name>hive.exec.local.scratchdir</name>
        <value>/opt/workspaces/data/hive</value>
    </property>
    <property>
        <name>hive.downloaded.resources.dir</name>
        <value>/opt/workspaces/data/hive</value>
    </property>
    <property>
        <name>hive.querylog.location</name>
        <value>/opt/workspaces/data/hive</value>
    </property>
    <property>
        <name>hive.server2.logging.operation.log.location</name>
        <value>/opt/workspaces/data/hive</value>
    </property>
    <property>
     <name>hive.querylog.location</name>
     <value>/opt/workspaces/frameworks/logs/hive</value>
    </property>
    <property>
      <name>hive.metastore.schema.verification</name>
      <value>false</value>
    </property>
</configuration>
EOF

hive.log.dir=/opt/workspaces/frameworks/logs/hive

wget -P ./frameworks/hive/lib http://central.maven.org/maven2/mysql/mysql-connector-java/5.1.36/mysql-connector-java-5.1.36.jar

nohup ./frameworks/hive/bin/hive --service metastore >/dev/null 2>&1 &

cat>test_person.txt<<EOF
tom 20
jack    24
nestor  29
EOF
./frameworks/hadoop/bin/hdfs dfs -mkdir /tmp/hivetest/
./frameworks/hadoop/bin/hdfs dfs -put ./test_person.txt /tmp/hivetest/
./frameworks/hive/bin/hive
CREATE EXTERNAL TABLE test_person(name STRING,age INT)ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' LOCATION '/tmp/hivetest';
select * from test_person;

# 切换到工作空间
cd /opt/workspaces
# 创建Kafka数据目录
mkdir data/kafka
# 创建Kafka日志目录
mkdir logs/kafka

wget http://mirrors.cnnic.cn/apache/kafka/0.8.2.2/kafka_2.10-0.8.2.2.tgz
tar -zxf kafka_2.10-0.8.2.2.tgz
rm -rf kafka_2.10-0.8.2.2.tgz
mv kafka_2.10-0.8.2.2 ./frameworks/kafka

# 设置如下参数：
broker.id=0
port=9092
# 对外主机名
advertised.host.name=<Docker对外IP>
# 数据目录
log.dirs=/opt/workspaces/data/kafka
# 连接ZK
zookeeper.connect=localhost:2181

# 设置日志目录
kafka.logs.dir=/opt/workspaces/logs/kafka

nohup ./frameworks/kafka/bin/kafka-server-start.sh ./frameworks/kafka/config/server.properties >/dev/null 2>&1 &

./frameworks/kafka/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test
./frameworks/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic test
> This is a message
> This is another message
# 在另个窗口输入
./frameworks/kafka/bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic test --from-beginning
> This is a message
> This is another message

wget http://mirror.bit.edu.cn/apache/sqoop/1.4.6/sqoop-1.4.6.bin__hadoop-2.0.4-alpha.tar.gz
tar -zxf sqoop-1.4.6.bin__hadoop-2.0.4-alpha.tar.gz
rm -rf sqoop-1.4.6.bin__hadoop-2.0.4-alpha.tar.gz
mv sqoop-1.4.6.bin__hadoop-2.0.4-alpha ./frameworks/sqoop

export SQOOP_HOME=/opt/workspaces/frameworks/sqoop

export HADOOP_COMMON_HOME=/opt/workspaces/frameworks/hadoop
export HADOOP_MAPRED_HOME=/opt/workspaces/frameworks/hadoop
export HIVE_HOME=/opt/workspaces/frameworks/hive

wget -P ./frameworks/sqoop/lib http://central.maven.org/maven2/mysql/mysql-connector-java/5.1.36/mysql-connector-java-5.1.36.jar

./frameworks/sqoop/bin/sqoop version

./frameworks/sqoop/bin/sqoop import --connect jdbc:mysql://<host>:<port>/hive \
 --username hive --password hive \
 --table ROLES \
 --where 1=1 \
 --hive-import --hive-table hive_role

# 如何要启用增量导入需要加上如下参数
--incremental lastmodified --check-column <source field> --last-value ''

wget http://mirrors.cnnic.cn/apache/incubator/zeppelin/0.5.6-incubating/zeppelin-0.5.6-incubating-bin-all.tgz
tar -zxf zeppelin-0.5.6-incubating-bin-all.tgz
rm -rf zeppelin-0.5.6-incubating-bin-all.tgz
mv zeppelin-0.5.6-incubating-bin-all ./frameworks/zeppelin

export ZEPPELIN_PORT=28081

export SPARK_HOME=/opt/workspaces/frameworks/spark
export HADOOP_CONF_DIR=/opt/workspaces/frameworks/hadoop
export SPARK_SUBMIT_OPTIONS="--packages com.databricks:spark-csv_2.10:1.2.0"

./frameworks/zeppelin/bin/zeppelin.sh start

官网：http://redis.io/
数据类型：http://blog.csdn.net/gaogaoshan/article/details/41039581