Operating system: CentOS Linux release 7.4.1708(Core)64 bit
The software used is as follows:
jdk1.8.0_112.tar.gz
hadoop-2.7.6-src.tar.gz
apache-ant-1.9.11-bin.tar.gz
apache-maven-3.5.3-bin.tar.gz
findbugs-3.0.1.tar.gz
protobuf-2.5.0.tar.gz
[ root@mini05 ~]# yum install -y cmake
[ root@mini05 ~]# yum install -y openssl-devel
[ root@mini05 ~]# yum install -y ncurses-devel
[ root@mini05 software]# pwd
/app/software
[ root@mini05 software]# tar xf jdk1.8.0_112.tar.gz
[ root@mini05 software]# ll
total 201392
drwxr-xr-x 8101434096 Dec 2013:27 jdk1.8.0_112
- rw-r--r--1 root root 189815615 Mar 1216:47 jdk1.8.0_112.tar.gz
[ root@mini05 software]# mv jdk1.8.0_112//app/[root@mini05 software]# cd /app/[root@mini05 app]# ln -s jdk1.8.0_112/ jdk
[ root@mini05 app]# ll
total 8
lrwxrwxrwx 1 root root 13 May 1623:19 jdk -> jdk1.8.0_112/
drwxr-xr-x 8101434096 Dec 2013:27 jdk1.8.0_112
[ root@mini05 ~]$ pwd
/app
[ root@mini05 ~]$ ll -d jdk* #You can choose the jdk version according to the actual situation, among which jdk1.8 can be compatible with jdk1.7
lrwxrwxrwx 1 yun yun 11 Mar 1514:58 jdk -> jdk1.8.0_112
drwxr-xr-x 8 yun yun 4096 Dec 2013:27 jdk1.8.0_112
[ root@mini05 profile.d]$ pwd
/etc/profile.d
[ root@mini05 profile.d]$ cat jdk.sh #java environment variables
export JAVA_HOME=/app/jdk
export JRE_HOME=/app/jdk/jre
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar:$JRE_HOME/lib:$CLASSPATH
export PATH=$JAVA_HOME/bin:$PATH
[ root@mini05 profile.d]# source /etc/profile
[ root@mini05 profile.d]$ java -version
java version "1.8.0_112"Java(TM) SE Runtime Environment(build 1.8.0_112-b15)
Java HotSpot(TM)64-Bit Server VM(build 25.112-b15, mixed mode)
[ root@mini05 software]# pwd
/app/software
[ root@mini05 software]# tar xf apache-maven-3.5.3-bin.tar.gz
[ root@mini05 software]# mv apache-maven-3.5.3/app/[root@mini05 software]# cd /app/[root@mini05 app]# ln -s apache-maven-3.5.3 maven
[ root@mini05 app]# ll
total 0
drwxr-xr-x 6 root root 99 Jun 810:18 apache-maven-3.5.3
lrwxrwxrwx 1 yun yun 13 Jun 722:49 jdk -> jdk1.8.0_112/
drwxr-xr-x 8 yun yun 255 Sep 232016 jdk1.8.0_112
lrwxrwxrwx 1 root root 18 Jun 810:19 maven -> apache-maven-3.5.3
drwxrwxr-x 2 yun yun 222 Jun 810:18 software
[ root@mini05 profile.d]# pwd
/etc/profile.d
[ root@mini05 profile.d]# cat maven.sh
export MAVEN_HOME=/app/maven/export PATH=$MAVEN_HOME/bin:$PATH
[ root@mini05 profile.d]# source /etc/profile
[ root@mini05 profile.d]# mvn -v
Apache Maven 3.5.3(3383c37e1f9e9b3bc3df5050c29c8aff9f295297;2018-02-25T03:49:05+08:00)
Maven home:/app/maven
Java version:1.8.0_112, vendor: Oracle Corporation
Java home:/app/jdk1.8.0_112/jre
Default locale: en_US, platform encoding: UTF-8
OS name:"linux", version:"3.10.0-693.el7.x86_64", arch:"amd64", family:"unix"
[ root@mini05 software]# pwd
/app/software
[ root@mini05 software]# tar xf apache-ant-1.9.11-bin.tar.gz
[ root@mini05 software]# mv apache-ant-1.9.11/app/[root@mini05 app]# ln -s apache-ant-1.9.11 ant
[ root@mini05 app]# ll
total 0
lrwxrwxrwx 1 root root 17 Jun 810:38 ant -> apache-ant-1.9.11
drwxr-xr-x 6 root root 235 Mar 2401:08 apache-ant-1.9.11
drwxr-xr-x 6 root root 99 Jun 810:18 apache-maven-3.5.3
lrwxrwxrwx 1 yun yun 13 Jun 722:49 jdk -> jdk1.8.0_112/
drwxr-xr-x 8 yun yun 255 Sep 232016 jdk1.8.0_112
lrwxrwxrwx 1 root root 18 Jun 810:19 maven -> apache-maven-3.5.3
drwxrwxr-x 2 yun yun 222 Jun 810:18 software
[ root@mini05 profile.d]# pwd
/etc/profile.d
[ root@mini05 profile.d]# vim ant.sh
export ANT_HOME="/app/ant"export PATH=$ANT_HOME/bin:$PATH
[ root@mini05 profile.d]# source /etc/profile
[ root@mini05 profile.d]# ant -version
Apache Ant(TM) version 1.9.11 compiled on March 232018
[ root@mini05 software]# pwd
/app/software
[ root@mini05 software]# tar xf findbugs-3.0.1.tar.gz
[ root@mini05 software]# mv findbugs-3.0.1/app/[root@mini05 software]# cd /app/[root@mini05 app]# ln -s findbugs-3.0.1/ findbugs
[ root@mini05 app]# ll
total 0
lrwxrwxrwx 1 root root 17 Jun 810:38 ant -> apache-ant-1.9.11
drwxr-xr-x 6 root root 235 Mar 2401:08 apache-ant-1.9.11
drwxr-xr-x 6 root root 99 Jun 810:18 apache-maven-3.5.3
lrwxrwxrwx 1 root root 15 Jun 811:01 findbugs -> findbugs-3.0.1/
drwxr-xr-x 8 root root 104 Jun 811:00 findbugs-3.0.1
lrwxrwxrwx 1 yun yun 13 Jun 722:49 jdk -> jdk1.8.0_112/
drwxr-xr-x 8 yun yun 255 Sep 232016 jdk1.8.0_112
lrwxrwxrwx 1 root root 18 Jun 810:19 maven -> apache-maven-3.5.3
drwxrwxr-x 2 yun yun 222 Jun 811:01 software
[ root@mini05 profile.d]# pwd
/etc/profile.d
[ root@mini05 profile.d]# cat findbugs.sh
export FINDBUGS_HOME="/app/findbugs"export PATH=$FINDBUGS_HOME/bin:$PATH
[ root@mini05 profile.d]# source /etc/profile
[ root@mini05 profile.d]# findbugs -version
3.0.1
# Cannot use protobuf-2.6.1.tar.gz will go wrong
[ root@mini05 software]# pwd
/app/software
[ root@mini05 software]# tar xf protobuf-2.5.0.tar.gz
[ root@mini05 software]# cd protobuf-2.5.0/[root@mini05 protobuf-2.5.0]# pwd
/app/software/protobuf-2.5.0[root@mini05 protobuf-2.6.1]# ./configure
………………
[ root@mini05 protobuf-2.6.1]# make
………………
[ root@mini05 protobuf-2.6.1]# make install
………………
[ root@mini05 protobuf-2.6.1]# protoc --version
libprotoc 2.5.0
[ root@mini05 software]# pwd
/app/software
[ root@mini05 software]# tar xf hadoop-2.7.6-src.tar.gz
[ root@mini05 software]# cd hadoop-2.7.6-src/[root@mini05 hadoop-2.7.6-src]# pwd
/app/software/hadoop-2.7.6-src
[ root@mini05 hadoop-2.7.6-src]# mvn clean install -DskipTests #The clean parameter can be omitted
………………
### It lasts for a long time, and there may be a card owner in the middle, if the card owner then Ctrl+Just execute it after C. There may be multiple times in the overall process, and then execute the command after interruption.
………………………………
[ INFO]------------------------------------------------------------------------[INFO] Reactor Summary:[INFO][INFO] Apache Hadoop Main 2.7.6........................... SUCCESS [1.098 s][INFO] Apache Hadoop Build Tools .......................... SUCCESS [1.151 s][INFO] Apache Hadoop Project POM .......................... SUCCESS [0.851 s][INFO] Apache Hadoop Annotations .......................... SUCCESS [0.390 s][INFO] Apache Hadoop Project Dist POM ..................... SUCCESS [0.298 s][INFO] Apache Hadoop Assemblies ........................... SUCCESS [0.158 s][INFO] Apache Hadoop Maven Plugins ........................ SUCCESS [1.494 s][INFO] Apache Hadoop MiniKDC .............................. SUCCESS [3.178 s][INFO] Apache Hadoop Auth ................................. SUCCESS [1.902 s][INFO] Apache Hadoop Auth Examples ........................ SUCCESS [0.916 s][INFO] Apache Hadoop Common ............................... SUCCESS [38.980 s][INFO] Apache Hadoop NFS .................................. SUCCESS [1.097 s][INFO] Apache Hadoop KMS .................................. SUCCESS [13.255 s][INFO] Apache Hadoop Common Project ....................... SUCCESS [0.099 s][INFO] Apache Hadoop HDFS ................................. SUCCESS [57.047 s][INFO] Apache Hadoop HttpFS ............................... SUCCESS [7.958 s][INFO] Apache Hadoop HDFS BookKeeper Journal .............. SUCCESS [49.933 s][INFO] Apache Hadoop HDFS-NFS ............................. SUCCESS [0.632 s][INFO] Apache Hadoop HDFS Project ......................... SUCCESS [0.048 s][INFO] hadoop-yarn ........................................ SUCCESS [0.045 s][INFO] hadoop-yarn-api .................................... SUCCESS [3.861 s][INFO] hadoop-yarn-common ................................. SUCCESS [01:35 min][INFO] hadoop-yarn-server ................................. SUCCESS [0.030 s][INFO] hadoop-yarn-server-common .......................... SUCCESS [1.002 s][INFO] hadoop-yarn-server-nodemanager ..................... SUCCESS [1.825 s][INFO] hadoop-yarn-server-web-proxy ....................... SUCCESS [0.499 s][INFO] hadoop-yarn-server-applicationhistoryservice ....... SUCCESS [0.940 s][INFO] hadoop-yarn-server-resourcemanager ................. SUCCESS [3.872 s][INFO] hadoop-yarn-server-tests ........................... SUCCESS [0.759 s][INFO] hadoop-yarn-client ................................. SUCCESS [0.687 s][INFO] hadoop-yarn-server-sharedcachemanager .............. SUCCESS [0.850 s][INFO] hadoop-yarn-applications ........................... SUCCESS [0.034 s][INFO] hadoop-yarn-applications-distributedshell .......... SUCCESS [0.396 s][INFO] hadoop-yarn-applications-unmanaged-am-launcher ..... SUCCESS [0.272 s][INFO] hadoop-yarn-site ................................... SUCCESS [0.040 s][INFO] hadoop-yarn-registry ............................... SUCCESS [0.711 s][INFO] hadoop-yarn-project ................................ SUCCESS [0.248 s][INFO] hadoop-mapreduce-client ............................ SUCCESS [0.114 s][INFO] hadoop-mapreduce-client-core ....................... SUCCESS [2.589 s][INFO] hadoop-mapreduce-client-common ..................... SUCCESS [1.546 s][INFO] hadoop-mapreduce-client-shuffle .................... SUCCESS [0.467 s][INFO] hadoop-mapreduce-client-app ........................ SUCCESS [1.305 s][INFO] hadoop-mapreduce-client-hs ......................... SUCCESS [0.780 s][INFO] hadoop-mapreduce-client-jobclient .................. SUCCESS [19.947 s][INFO] hadoop-mapreduce-client-hs-plugins ................. SUCCESS [0.364 s][INFO] Apache Hadoop MapReduce Examples ................... SUCCESS [0.831 s][INFO] hadoop-mapreduce ................................... SUCCESS [0.169 s][INFO] Apache Hadoop MapReduce Streaming .................. SUCCESS [8.894 s][INFO] Apache Hadoop Distributed Copy ..................... SUCCESS [34.396 s][INFO] Apache Hadoop Archives ............................. SUCCESS [0.309 s][INFO] Apache Hadoop Rumen ................................ SUCCESS [0.471 s][INFO] Apache Hadoop Gridmix .............................. SUCCESS [0.484 s][INFO] Apache Hadoop Data Join ............................ SUCCESS [0.187 s][INFO] Apache Hadoop Ant Tasks ............................ SUCCESS [0.099 s][INFO] Apache Hadoop Extras ............................... SUCCESS [0.386 s][INFO] Apache Hadoop Pipes ................................ SUCCESS [0.024 s][INFO] Apache Hadoop OpenStack support .................... SUCCESS [0.445 s][INFO] Apache Hadoop Amazon Web Services support .......... SUCCESS [03:23 min][INFO] Apache Hadoop Azure support ........................ SUCCESS [17.180 s][INFO] Apache Hadoop Client ............................... SUCCESS [1.042 s][INFO] Apache Hadoop Mini-Cluster ......................... SUCCESS [0.862 s][INFO] Apache Hadoop Scheduler Load Simulator ............. SUCCESS [1.292 s][INFO] Apache Hadoop Tools Dist ........................... SUCCESS [0.614 s][INFO] Apache Hadoop Tools ................................ SUCCESS [0.033 s][INFO] Apache Hadoop Distribution 2.7.6................... SUCCESS [0.124 s][INFO]------------------------------------------------------------------------[INFO] BUILD SUCCESS
[ INFO]------------------------------------------------------------------------[INFO] Total time:09:51 min
[ INFO] Finished at:2018-06-08T16:28:47+08:00[INFO]------------------------------------------------------------------------[root@mini05 hadoop-2.7.6-src]# mvn package-Pdist,native -DskipTests -Dtar
…………………………
[ INFO] Executing tasks
main:[exec] $ tar cf hadoop-2.7.6.tar hadoop-2.7.6[exec] $ gzip -f hadoop-2.7.6.tar
[ exec][exec] Hadoop dist tar available at:/app/software/hadoop-2.7.6-src/hadoop-dist/target/hadoop-2.7.6.tar.gz #Hadoop compiled jar package location
[ exec][INFO] Executed tasks
[ INFO][INFO]--- maven-javadoc-plugin:2.8.1:jar(module-javadocs) @ hadoop-dist ---[INFO] Building jar:/app/software/hadoop-2.7.6-src/hadoop-dist/target/hadoop-dist-2.7.6-javadoc.jar
[ INFO]------------------------------------------------------------------------[INFO] Reactor Summary:[INFO][INFO] Apache Hadoop Main 2.7.6........................... SUCCESS [7.433 s][INFO] Apache Hadoop Build Tools .......................... SUCCESS [0.789 s][INFO] Apache Hadoop Project POM .......................... SUCCESS [21.333 s][INFO] Apache Hadoop Annotations .......................... SUCCESS [1.470 s][INFO] Apache Hadoop Assemblies ........................... SUCCESS [0.135 s][INFO] Apache Hadoop Project Dist POM ..................... SUCCESS [13.132 s][INFO] Apache Hadoop Maven Plugins ........................ SUCCESS [2.676 s][INFO] Apache Hadoop MiniKDC .............................. SUCCESS [3.423 s][INFO] Apache Hadoop Auth ................................. SUCCESS [3.391 s][INFO] Apache Hadoop Auth Examples ........................ SUCCESS [2.545 s][INFO] Apache Hadoop Common ............................... SUCCESS [01:06 min][INFO] Apache Hadoop NFS .................................. SUCCESS [3.635 s][INFO] Apache Hadoop KMS .................................. SUCCESS [9.051 s][INFO] Apache Hadoop Common Project ....................... SUCCESS [0.044 s][INFO] Apache Hadoop HDFS ................................. SUCCESS [01:17 min][INFO] Apache Hadoop HttpFS ............................... SUCCESS [12.516 s][INFO] Apache Hadoop HDFS BookKeeper Journal .............. SUCCESS [9.920 s][INFO] Apache Hadoop HDFS-NFS ............................. SUCCESS [2.311 s][INFO] Apache Hadoop HDFS Project ......................... SUCCESS [0.029 s][INFO] hadoop-yarn ........................................ SUCCESS [0.036 s][INFO] hadoop-yarn-api .................................... SUCCESS [22.028 s][INFO] hadoop-yarn-common ................................. SUCCESS [16.103 s][INFO] hadoop-yarn-server ................................. SUCCESS [0.041 s][INFO] hadoop-yarn-server-common .......................... SUCCESS [6.938 s][INFO] hadoop-yarn-server-nodemanager ..................... SUCCESS [10.326 s][INFO] hadoop-yarn-server-web-proxy ....................... SUCCESS [2.036 s][INFO] hadoop-yarn-server-applicationhistoryservice ....... SUCCESS [4.360 s][INFO] hadoop-yarn-server-resourcemanager ................. SUCCESS [11.051 s][INFO] hadoop-yarn-server-tests ........................... SUCCESS [3.508 s][INFO] hadoop-yarn-client ................................. SUCCESS [3.938 s][INFO] hadoop-yarn-server-sharedcachemanager .............. SUCCESS [2.340 s][INFO] hadoop-yarn-applications ........................... SUCCESS [0.022 s][INFO] hadoop-yarn-applications-distributedshell .......... SUCCESS [1.674 s][INFO] hadoop-yarn-applications-unmanaged-am-launcher ..... SUCCESS [1.333 s][INFO] hadoop-yarn-site ................................... SUCCESS [0.020 s][INFO] hadoop-yarn-registry ............................... SUCCESS [3.027 s][INFO] hadoop-yarn-project ................................ SUCCESS [2.675 s][INFO] hadoop-mapreduce-client ............................ SUCCESS [0.086 s][INFO] hadoop-mapreduce-client-core ....................... SUCCESS [11.176 s][INFO] hadoop-mapreduce-client-common ..................... SUCCESS [10.672 s][INFO] hadoop-mapreduce-client-shuffle .................... SUCCESS [2.336 s][INFO] hadoop-mapreduce-client-app ........................ SUCCESS [5.157 s][INFO] hadoop-mapreduce-client-hs ......................... SUCCESS [3.429 s][INFO] hadoop-mapreduce-client-jobclient .................. SUCCESS [2.947 s][INFO] hadoop-mapreduce-client-hs-plugins ................. SUCCESS [1.182 s][INFO] Apache Hadoop MapReduce Examples ................... SUCCESS [3.813 s][INFO] hadoop-mapreduce ................................... SUCCESS [1.939 s][INFO] Apache Hadoop MapReduce Streaming .................. SUCCESS [2.445 s][INFO] Apache Hadoop Distributed Copy ..................... SUCCESS [5.319 s][INFO] Apache Hadoop Archives ............................. SUCCESS [1.183 s][INFO] Apache Hadoop Rumen ................................ SUCCESS [3.061 s][INFO] Apache Hadoop Gridmix .............................. SUCCESS [2.350 s][INFO] Apache Hadoop Data Join ............................ SUCCESS [1.619 s][INFO] Apache Hadoop Ant Tasks ............................ SUCCESS [1.436 s][INFO] Apache Hadoop Extras ............................... SUCCESS [1.817 s][INFO] Apache Hadoop Pipes ................................ SUCCESS [5.664 s][INFO] Apache Hadoop OpenStack support .................... SUCCESS [2.720 s][INFO] Apache Hadoop Amazon Web Services support .......... SUCCESS [2.630 s][INFO] Apache Hadoop Azure support ........................ SUCCESS [2.453 s][INFO] Apache Hadoop Client ............................... SUCCESS [6.406 s][INFO] Apache Hadoop Mini-Cluster ......................... SUCCESS [0.674 s][INFO] Apache Hadoop Scheduler Load Simulator ............. SUCCESS [3.096 s][INFO] Apache Hadoop Tools Dist ........................... SUCCESS [6.273 s][INFO] Apache Hadoop Tools ................................ SUCCESS [0.032 s][INFO] Apache Hadoop Distribution 2.7.6................... SUCCESS [25.962 s][INFO]------------------------------------------------------------------------[INFO] BUILD SUCCESS
[ INFO]------------------------------------------------------------------------[INFO] Total time:07:28 min
[ INFO] Finished at:2018-06-08T16:37:06+08:00[INFO]------------------------------------------------------------------------
[ root@mini05 native]# pwd
/app/software/hadoop-2.7.6-src/hadoop-dist/target/hadoop-2.7.6/lib/native
[ root@mini05 native]# file *
libhadoop.a: current ar archive
libhadooppipes.a: current ar archive
libhadoop.so: symbolic link to `libhadoop.so.1.0.0'
libhadoop.so.1.0.0: ELF 64-bit LSB shared object, x86-64, version 1 (SYSV), dynamically linked, BuildID[sha1]=1092bb61838f0b1c0d982b20dc8223ae93ba708f, not stripped
libhadooputils.a: current ar archive
libhdfs.a: current ar archive
libhdfs.so: symbolic link to `libhdfs.so.0.0.0'
libhdfs.so.0.0.0: ELF 64-bit LSB shared object, x86-64, version 1(SYSV), dynamically linked, BuildID[sha1]=e719718169b5319c4f9109b5b5d8f33cebe65ed6, not stripped
[ root@mini05 target]# pwd
/app/software/hadoop-2.7.6-src/hadoop-dist/target
[ root@mini05 target]# ll -h
total 571M
drwxr-xr-x 2 root root 28 Jun 816:28 antrun
drwxr-xr-x 3 root root 22 Jun 816:28 classes
- rw-r--r--1 root root 1.9K Jun 816:36 dist-layout-stitching.sh
- rw-r--r--1 root root 643 Jun 816:36 dist-tar-stitching.sh
drwxr-xr-x 9 root root 149 Jun 816:36 hadoop-2.7.6-rw-r--r--1 root root 190M Jun 816:36 hadoop-2.7.6.tar.gz
- rw-r--r--1 root root 26K Jun 816:36 hadoop-dist-2.7.6.jar
- rw-r--r--1 root root 381M Jun 816:37 hadoop-dist-2.7.6-javadoc.jar
- rw-r--r--1 root root 24K Jun 816:36 hadoop-dist-2.7.6-sources.jar
- rw-r--r--1 root root 24K Jun 816:36 hadoop-dist-2.7.6-test-sources.jar
drwxr-xr-x 2 root root 51 Jun 816:36 javadoc-bundle-options
drwxr-xr-x 2 root root 28 Jun 816:28 maven-archiver
drwxr-xr-x 3 root root 22 Jun 816:28 maven-shared-archive-resources
drwxr-xr-x 3 root root 22 Jun 816:28 test-classes
drwxr-xr-x 2 root root 6 Jun 816:28 test-dir
PS：
1 The software required for related compilation is placed in the cloud disk, in which CentOS-7.4_hadoop-2.7.6.tar.gz is hadoop compiled above-2.7.6.tar.Rename by gz can be used directly.
23 Link: https://pan.baidu.com/s/1saAbQdoO4GRIbdCH--prkw password: 372f
1、 hadoop2.7.3 compile and install
2、 Why is it best to recompile the source code when building hadoop
Recommended Posts