From 97c6ef0da9073050c6a539bce86ab63550bb617c Mon Sep 17 00:00:00 2001 From: zk Date: Wed, 5 Oct 2022 19:51:50 +0800 Subject: [PATCH 1/6] fix: network typo --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f95859a6..3da27d44 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -DOCKER_NETWORK = dockerhadoop_default +DOCKER_NETWORK = docker-hadoop_default ENV_FILE = hadoop.env current_branch := $(shell git rev-parse --abbrev-ref HEAD) build: From 1782b5ee335fdea262df42544d383d03f8f413e7 Mon Sep 17 00:00:00 2001 From: zk Date: Wed, 5 Oct 2022 20:03:54 +0800 Subject: [PATCH 2/6] feat: export port --- docker-compose.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index f4f87184..33dfba31 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,6 +8,8 @@ services: - hadoop_namenode:/hadoop/dfs/name environment: - CLUSTER_NAME=test + ports: + - 50070:50070 env_file: - ./hadoop.env @@ -18,6 +20,8 @@ services: - hadoop_datanode:/hadoop/dfs/data environment: SERVICE_PRECONDITION: "namenode:50070" + ports: + - 50075:50075 env_file: - ./hadoop.env @@ -26,6 +30,8 @@ services: container_name: resourcemanager environment: SERVICE_PRECONDITION: "namenode:50070 datanode:50075" + ports: + - 8088:8088 env_file: - ./hadoop.env From 1843107b6489386f29f49d9aedf1f8dc7118095a Mon Sep 17 00:00:00 2001 From: zk Date: Wed, 5 Oct 2022 23:19:05 +0800 Subject: [PATCH 3/6] fix: expose 9000 for host java client in intellij --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index 33dfba31..541385f5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,6 +10,7 @@ services: - CLUSTER_NAME=test ports: - 50070:50070 + - 9000:9000 env_file: - ./hadoop.env From bc9ab7a15b997e091b5ef5103e5e1ebcac1a6321 Mon Sep 17 00:00:00 2001 From: zk Date: Wed, 5 Oct 2022 23:19:14 +0800 Subject: [PATCH 4/6] feat: simple test for java client --- java-src/hadoopdemo/hadoopdemo.iml | 2 + java-src/hadoopdemo/pom.xml | 83 +++++++++++++++++++ .../hadoopdemo/src/main/java/com/zk/App.java | 13 +++ .../src/main/resources/core-site.xml | 25 ++++++ .../src/main/resources/hdfs-site.xml | 31 +++++++ .../src/test/java/com/zk/AppTest.java | 35 ++++++++ 6 files changed, 189 insertions(+) create mode 100644 java-src/hadoopdemo/hadoopdemo.iml create mode 100644 java-src/hadoopdemo/pom.xml create mode 100644 java-src/hadoopdemo/src/main/java/com/zk/App.java create mode 100644 java-src/hadoopdemo/src/main/resources/core-site.xml create mode 100644 java-src/hadoopdemo/src/main/resources/hdfs-site.xml create mode 100644 java-src/hadoopdemo/src/test/java/com/zk/AppTest.java diff --git a/java-src/hadoopdemo/hadoopdemo.iml b/java-src/hadoopdemo/hadoopdemo.iml new file mode 100644 index 00000000..78b2cc53 --- /dev/null +++ b/java-src/hadoopdemo/hadoopdemo.iml @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/java-src/hadoopdemo/pom.xml b/java-src/hadoopdemo/pom.xml new file mode 100644 index 00000000..8a73a993 --- /dev/null +++ b/java-src/hadoopdemo/pom.xml @@ -0,0 +1,83 @@ + + + + 4.0.0 + + com.zk + hadoop-demo + 1.0-SNAPSHOT + + hadoop-demo + + http://www.example.com + + + UTF-8 + + + + + + org.junit.jupiter + junit-jupiter + 5.8.2 + + + org.apache.hadoop + hadoop-common + 2.7.5 + + + org.apache.hadoop + hadoop-hdfs + 2.7.5 + + + + + + + + + maven-clean-plugin + 3.1.0 + + + + maven-resources-plugin + 3.0.2 + + + maven-compiler-plugin + 3.8.0 + + + maven-surefire-plugin + 2.22.1 + + + maven-jar-plugin + 3.0.2 + + + maven-install-plugin + 2.5.2 + + + maven-deploy-plugin + 2.8.2 + + + + maven-site-plugin + 3.7.1 + + + maven-project-info-reports-plugin + 3.0.0 + + + + + diff --git a/java-src/hadoopdemo/src/main/java/com/zk/App.java b/java-src/hadoopdemo/src/main/java/com/zk/App.java new file mode 100644 index 00000000..377b243b --- /dev/null +++ b/java-src/hadoopdemo/src/main/java/com/zk/App.java @@ -0,0 +1,13 @@ +package com.zk; + +/** + * Hello world! + * + */ +public class App +{ + public static void main( String[] args ) + { + System.out.println( "Hello World!" ); + } +} diff --git a/java-src/hadoopdemo/src/main/resources/core-site.xml b/java-src/hadoopdemo/src/main/resources/core-site.xml new file mode 100644 index 00000000..4d867e23 --- /dev/null +++ b/java-src/hadoopdemo/src/main/resources/core-site.xml @@ -0,0 +1,25 @@ + + + + + + + + hadoop.proxyuser.hue.hosts* + fs.defaultFShdfs://namenode:9000 + io.compression.codecsorg.apache.hadoop.io.compress.SnappyCodec + hadoop.proxyuser.hue.groups* + hadoop.http.staticuser.userroot + \ No newline at end of file diff --git a/java-src/hadoopdemo/src/main/resources/hdfs-site.xml b/java-src/hadoopdemo/src/main/resources/hdfs-site.xml new file mode 100644 index 00000000..92dcc353 --- /dev/null +++ b/java-src/hadoopdemo/src/main/resources/hdfs-site.xml @@ -0,0 +1,31 @@ + + + + + + + + + dfs.namenode.name.dirfile:///hadoop/dfs/name + dfs.namenode.datanode.registration.ip-hostname-checkfalse + dfs.permissions.enabledfalse + dfs.webhdfs.enabledtrue + dfs.namenode.rpc-bind-host0.0.0.0 + dfs.namenode.servicerpc-bind-host0.0.0.0 + dfs.namenode.http-bind-host0.0.0.0 + dfs.namenode.https-bind-host0.0.0.0 + dfs.client.use.datanode.hostnametrue + dfs.datanode.use.datanode.hostnametrue + \ No newline at end of file diff --git a/java-src/hadoopdemo/src/test/java/com/zk/AppTest.java b/java-src/hadoopdemo/src/test/java/com/zk/AppTest.java new file mode 100644 index 00000000..c09e322e --- /dev/null +++ b/java-src/hadoopdemo/src/test/java/com/zk/AppTest.java @@ -0,0 +1,35 @@ +package com.zk; + + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.IOException; + +/** + * Unit test for simple App. + */ +public class AppTest { + + Configuration configuration = null; + FileSystem fs = null; + + @BeforeEach + void setUp() throws IOException { + + configuration = new Configuration(true); + fs = FileSystem.get(configuration); + } + + @Test + public void mkdir() throws IOException { + Path path = new Path("/zk"); + if (!fs.exists(path)) { + fs.delete(path, true); + } + fs.mkdirs(path); + } +} From 3b069c870361be28d1254f7ac3d3c20c76cafa9f Mon Sep 17 00:00:00 2001 From: zk Date: Thu, 6 Oct 2022 01:27:28 +0800 Subject: [PATCH 5/6] fix: set hostname --- docker-compose.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 541385f5..d7f7071b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,6 +4,7 @@ services: namenode: image: bde2020/hadoop-namenode:2.0.0-hadoop2.7.4-java8 container_name: namenode + hostname: namenode volumes: - hadoop_namenode:/hadoop/dfs/name environment: @@ -17,18 +18,21 @@ services: datanode: image: bde2020/hadoop-datanode:2.0.0-hadoop2.7.4-java8 container_name: datanode + hostname: datanode volumes: - hadoop_datanode:/hadoop/dfs/data environment: SERVICE_PRECONDITION: "namenode:50070" ports: - 50075:50075 + - 50010:50010 env_file: - ./hadoop.env resourcemanager: image: bde2020/hadoop-resourcemanager:2.0.0-hadoop2.7.4-java8 container_name: resourcemanager + hostname: resourcemanager environment: SERVICE_PRECONDITION: "namenode:50070 datanode:50075" ports: @@ -39,6 +43,7 @@ services: nodemanager1: image: bde2020/hadoop-nodemanager:2.0.0-hadoop2.7.4-java8 container_name: nodemanager + hostname: nodemanager1 environment: SERVICE_PRECONDITION: "namenode:50070 datanode:50075 resourcemanager:8088" env_file: @@ -47,6 +52,7 @@ services: historyserver: image: bde2020/hadoop-historyserver:2.0.0-hadoop2.7.4-java8 container_name: historyserver + hostname: historyserver environment: SERVICE_PRECONDITION: "namenode:50070 datanode:50075 resourcemanager:8088" volumes: From 90d2f5d9e742b21edce4e1f3f01888da0b015207 Mon Sep 17 00:00:00 2001 From: zk Date: Thu, 6 Oct 2022 01:28:23 +0800 Subject: [PATCH 6/6] add upload demo adn hosts config --- README.md | 9 +++++++++ .../src/test/java/com/zk/AppTest.java | 20 ++++++++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b20a1ac0..0cc685fc 100644 --- a/README.md +++ b/README.md @@ -57,3 +57,12 @@ The available configurations are: * /etc/hadoop/mapred-site.xml MAPRED_CONF If you need to extend some other configuration file, refer to base/entrypoint.sh bash script. + + +set /etc/hosts +``` ini +# hadoop +127.0.0.1 namenode +127.0.0.1 datanode +``` + diff --git a/java-src/hadoopdemo/src/test/java/com/zk/AppTest.java b/java-src/hadoopdemo/src/test/java/com/zk/AppTest.java index c09e322e..11b493e1 100644 --- a/java-src/hadoopdemo/src/test/java/com/zk/AppTest.java +++ b/java-src/hadoopdemo/src/test/java/com/zk/AppTest.java @@ -2,12 +2,14 @@ import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IOUtils; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import java.io.IOException; +import java.io.*; /** * Unit test for simple App. @@ -32,4 +34,20 @@ public void mkdir() throws IOException { } fs.mkdirs(path); } + + + @Test + void upload() throws IOException { + BufferedInputStream bufferedInputStream = new BufferedInputStream(new FileInputStream(new File( + "./data/words.txt"))); + Path path = new Path("/zk/words2.txt"); + FSDataOutputStream fsDataOutputStream = fs.create(path); + IOUtils.copyBytes(bufferedInputStream,fsDataOutputStream,configuration, true); + } + + @Test + void read() { + Path path = new Path("/zk/words.txt"); + + } }