diff --git a/Makefile b/Makefile index f95859a6..3da27d44 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -DOCKER_NETWORK = dockerhadoop_default +DOCKER_NETWORK = docker-hadoop_default ENV_FILE = hadoop.env current_branch := $(shell git rev-parse --abbrev-ref HEAD) build: diff --git a/README.md b/README.md index b20a1ac0..0cc685fc 100644 --- a/README.md +++ b/README.md @@ -57,3 +57,12 @@ The available configurations are: * /etc/hadoop/mapred-site.xml MAPRED_CONF If you need to extend some other configuration file, refer to base/entrypoint.sh bash script. + + +set /etc/hosts +``` ini +# hadoop +127.0.0.1 namenode +127.0.0.1 datanode +``` + diff --git a/docker-compose.yml b/docker-compose.yml index f4f87184..d7f7071b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,34 +4,46 @@ services: namenode: image: bde2020/hadoop-namenode:2.0.0-hadoop2.7.4-java8 container_name: namenode + hostname: namenode volumes: - hadoop_namenode:/hadoop/dfs/name environment: - CLUSTER_NAME=test + ports: + - 50070:50070 + - 9000:9000 env_file: - ./hadoop.env datanode: image: bde2020/hadoop-datanode:2.0.0-hadoop2.7.4-java8 container_name: datanode + hostname: datanode volumes: - hadoop_datanode:/hadoop/dfs/data environment: SERVICE_PRECONDITION: "namenode:50070" + ports: + - 50075:50075 + - 50010:50010 env_file: - ./hadoop.env resourcemanager: image: bde2020/hadoop-resourcemanager:2.0.0-hadoop2.7.4-java8 container_name: resourcemanager + hostname: resourcemanager environment: SERVICE_PRECONDITION: "namenode:50070 datanode:50075" + ports: + - 8088:8088 env_file: - ./hadoop.env nodemanager1: image: bde2020/hadoop-nodemanager:2.0.0-hadoop2.7.4-java8 container_name: nodemanager + hostname: nodemanager1 environment: SERVICE_PRECONDITION: "namenode:50070 datanode:50075 resourcemanager:8088" env_file: @@ -40,6 +52,7 @@ services: historyserver: image: bde2020/hadoop-historyserver:2.0.0-hadoop2.7.4-java8 container_name: historyserver + hostname: historyserver environment: SERVICE_PRECONDITION: "namenode:50070 datanode:50075 resourcemanager:8088" volumes: diff --git a/java-src/hadoopdemo/hadoopdemo.iml b/java-src/hadoopdemo/hadoopdemo.iml new file mode 100644 index 00000000..78b2cc53 --- /dev/null +++ b/java-src/hadoopdemo/hadoopdemo.iml @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/java-src/hadoopdemo/pom.xml b/java-src/hadoopdemo/pom.xml new file mode 100644 index 00000000..8a73a993 --- /dev/null +++ b/java-src/hadoopdemo/pom.xml @@ -0,0 +1,83 @@ + + + + 4.0.0 + + com.zk + hadoop-demo + 1.0-SNAPSHOT + + hadoop-demo + + http://www.example.com + + + UTF-8 + + + + + + org.junit.jupiter + junit-jupiter + 5.8.2 + + + org.apache.hadoop + hadoop-common + 2.7.5 + + + org.apache.hadoop + hadoop-hdfs + 2.7.5 + + + + + + + + + maven-clean-plugin + 3.1.0 + + + + maven-resources-plugin + 3.0.2 + + + maven-compiler-plugin + 3.8.0 + + + maven-surefire-plugin + 2.22.1 + + + maven-jar-plugin + 3.0.2 + + + maven-install-plugin + 2.5.2 + + + maven-deploy-plugin + 2.8.2 + + + + maven-site-plugin + 3.7.1 + + + maven-project-info-reports-plugin + 3.0.0 + + + + + diff --git a/java-src/hadoopdemo/src/main/java/com/zk/App.java b/java-src/hadoopdemo/src/main/java/com/zk/App.java new file mode 100644 index 00000000..377b243b --- /dev/null +++ b/java-src/hadoopdemo/src/main/java/com/zk/App.java @@ -0,0 +1,13 @@ +package com.zk; + +/** + * Hello world! + * + */ +public class App +{ + public static void main( String[] args ) + { + System.out.println( "Hello World!" ); + } +} diff --git a/java-src/hadoopdemo/src/main/resources/core-site.xml b/java-src/hadoopdemo/src/main/resources/core-site.xml new file mode 100644 index 00000000..4d867e23 --- /dev/null +++ b/java-src/hadoopdemo/src/main/resources/core-site.xml @@ -0,0 +1,25 @@ + + + + + + + + hadoop.proxyuser.hue.hosts* + fs.defaultFShdfs://namenode:9000 + io.compression.codecsorg.apache.hadoop.io.compress.SnappyCodec + hadoop.proxyuser.hue.groups* + hadoop.http.staticuser.userroot + \ No newline at end of file diff --git a/java-src/hadoopdemo/src/main/resources/hdfs-site.xml b/java-src/hadoopdemo/src/main/resources/hdfs-site.xml new file mode 100644 index 00000000..92dcc353 --- /dev/null +++ b/java-src/hadoopdemo/src/main/resources/hdfs-site.xml @@ -0,0 +1,31 @@ + + + + + + + + + dfs.namenode.name.dirfile:///hadoop/dfs/name + dfs.namenode.datanode.registration.ip-hostname-checkfalse + dfs.permissions.enabledfalse + dfs.webhdfs.enabledtrue + dfs.namenode.rpc-bind-host0.0.0.0 + dfs.namenode.servicerpc-bind-host0.0.0.0 + dfs.namenode.http-bind-host0.0.0.0 + dfs.namenode.https-bind-host0.0.0.0 + dfs.client.use.datanode.hostnametrue + dfs.datanode.use.datanode.hostnametrue + \ No newline at end of file diff --git a/java-src/hadoopdemo/src/test/java/com/zk/AppTest.java b/java-src/hadoopdemo/src/test/java/com/zk/AppTest.java new file mode 100644 index 00000000..11b493e1 --- /dev/null +++ b/java-src/hadoopdemo/src/test/java/com/zk/AppTest.java @@ -0,0 +1,53 @@ +package com.zk; + + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IOUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.*; + +/** + * Unit test for simple App. + */ +public class AppTest { + + Configuration configuration = null; + FileSystem fs = null; + + @BeforeEach + void setUp() throws IOException { + + configuration = new Configuration(true); + fs = FileSystem.get(configuration); + } + + @Test + public void mkdir() throws IOException { + Path path = new Path("/zk"); + if (!fs.exists(path)) { + fs.delete(path, true); + } + fs.mkdirs(path); + } + + + @Test + void upload() throws IOException { + BufferedInputStream bufferedInputStream = new BufferedInputStream(new FileInputStream(new File( + "./data/words.txt"))); + Path path = new Path("/zk/words2.txt"); + FSDataOutputStream fsDataOutputStream = fs.create(path); + IOUtils.copyBytes(bufferedInputStream,fsDataOutputStream,configuration, true); + } + + @Test + void read() { + Path path = new Path("/zk/words.txt"); + + } +}