uboot: (firmwareOdroidC2/C4) don't invoke patch tool, use patches = [] instead
https://github.com/NixOS/nixpkgs/blob/master/pkgs/stdenv/generic/setup.sh#L948 this can do it nicely. Signed-off-by: Anton Arapov <anton@deadbeef.mx>
This commit is contained in:
Anton Arapov
Alan Daniels
commit
56de2bcd43
30691 changed files with 3076956 additions and 0 deletions
7
nixos/tests/hadoop/default.nix
Normal file
7
nixos/tests/hadoop/default.nix
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
{ handleTestOn, package, ... }:
|
||||
|
||||
{
|
||||
all = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./hadoop.nix { inherit package; };
|
||||
hdfs = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./hdfs.nix { inherit package; };
|
||||
yarn = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./yarn.nix { inherit package; };
|
||||
}
|
||||
255
nixos/tests/hadoop/hadoop.nix
Normal file
255
nixos/tests/hadoop/hadoop.nix
Normal file
|
|
@ -0,0 +1,255 @@
|
|||
# This test is very comprehensive. It tests whether all hadoop services work well with each other.
|
||||
# Run this when updating the Hadoop package or making significant changes to the hadoop module.
|
||||
# For a more basic test, see hdfs.nix and yarn.nix
|
||||
import ../make-test-python.nix ({ package, ... }: {
|
||||
name = "hadoop-combined";
|
||||
|
||||
nodes =
|
||||
let
|
||||
coreSite = {
|
||||
"fs.defaultFS" = "hdfs://ns1";
|
||||
};
|
||||
hdfsSite = {
|
||||
# HA Quorum Journal Manager configuration
|
||||
"dfs.nameservices" = "ns1";
|
||||
"dfs.ha.namenodes.ns1" = "nn1,nn2";
|
||||
"dfs.namenode.shared.edits.dir.ns1" = "qjournal://jn1:8485;jn2:8485;jn3:8485/ns1";
|
||||
"dfs.namenode.rpc-address.ns1.nn1" = "nn1:8020";
|
||||
"dfs.namenode.rpc-address.ns1.nn2" = "nn2:8020";
|
||||
"dfs.namenode.servicerpc-address.ns1.nn1" = "nn1:8022";
|
||||
"dfs.namenode.servicerpc-address.ns1.nn2" = "nn2:8022";
|
||||
"dfs.namenode.http-address.ns1.nn1" = "nn1:9870";
|
||||
"dfs.namenode.http-address.ns1.nn2" = "nn2:9870";
|
||||
|
||||
# Automatic failover configuration
|
||||
"dfs.client.failover.proxy.provider.ns1" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider";
|
||||
"dfs.ha.automatic-failover.enabled.ns1" = "true";
|
||||
"dfs.ha.fencing.methods" = "shell(true)";
|
||||
"ha.zookeeper.quorum" = "zk1:2181";
|
||||
};
|
||||
yarnSite = {
|
||||
"yarn.resourcemanager.zk-address" = "zk1:2181";
|
||||
"yarn.resourcemanager.ha.enabled" = "true";
|
||||
"yarn.resourcemanager.ha.rm-ids" = "rm1,rm2";
|
||||
"yarn.resourcemanager.hostname.rm1" = "rm1";
|
||||
"yarn.resourcemanager.hostname.rm2" = "rm2";
|
||||
"yarn.resourcemanager.ha.automatic-failover.enabled" = "true";
|
||||
"yarn.resourcemanager.cluster-id" = "cluster1";
|
||||
# yarn.resourcemanager.webapp.address needs to be defined even though yarn.resourcemanager.hostname is set. This shouldn't be necessary, but there's a bug in
|
||||
# hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/amfilter/AmFilterInitializer.java:70
|
||||
# that causes AM containers to fail otherwise.
|
||||
"yarn.resourcemanager.webapp.address.rm1" = "rm1:8088";
|
||||
"yarn.resourcemanager.webapp.address.rm2" = "rm2:8088";
|
||||
};
|
||||
in
|
||||
{
|
||||
zk1 = { ... }: {
|
||||
services.zookeeper.enable = true;
|
||||
networking.firewall.allowedTCPPorts = [ 2181 ];
|
||||
};
|
||||
|
||||
# HDFS cluster
|
||||
nn1 = { ... }: {
|
||||
services.hadoop = {
|
||||
inherit package coreSite hdfsSite;
|
||||
hdfs.namenode = {
|
||||
enable = true;
|
||||
openFirewall = true;
|
||||
};
|
||||
hdfs.zkfc.enable = true;
|
||||
};
|
||||
};
|
||||
nn2 = { ... }: {
|
||||
services.hadoop = {
|
||||
inherit package coreSite hdfsSite;
|
||||
hdfs.namenode = {
|
||||
enable = true;
|
||||
openFirewall = true;
|
||||
};
|
||||
hdfs.zkfc.enable = true;
|
||||
};
|
||||
};
|
||||
|
||||
jn1 = { ... }: {
|
||||
services.hadoop = {
|
||||
inherit package coreSite hdfsSite;
|
||||
hdfs.journalnode = {
|
||||
enable = true;
|
||||
openFirewall = true;
|
||||
};
|
||||
};
|
||||
};
|
||||
jn2 = { ... }: {
|
||||
services.hadoop = {
|
||||
inherit package coreSite hdfsSite;
|
||||
hdfs.journalnode = {
|
||||
enable = true;
|
||||
openFirewall = true;
|
||||
};
|
||||
};
|
||||
};
|
||||
jn3 = { ... }: {
|
||||
services.hadoop = {
|
||||
inherit package coreSite hdfsSite;
|
||||
hdfs.journalnode = {
|
||||
enable = true;
|
||||
openFirewall = true;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
dn1 = { ... }: {
|
||||
services.hadoop = {
|
||||
inherit package coreSite hdfsSite;
|
||||
hdfs.datanode = {
|
||||
enable = true;
|
||||
openFirewall = true;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
# YARN cluster
|
||||
rm1 = { options, ... }: {
|
||||
services.hadoop = {
|
||||
inherit package coreSite hdfsSite yarnSite;
|
||||
yarn.resourcemanager = {
|
||||
enable = true;
|
||||
openFirewall = true;
|
||||
};
|
||||
};
|
||||
};
|
||||
rm2 = { options, ... }: {
|
||||
services.hadoop = {
|
||||
inherit package coreSite hdfsSite yarnSite;
|
||||
yarn.resourcemanager = {
|
||||
enable = true;
|
||||
openFirewall = true;
|
||||
};
|
||||
};
|
||||
};
|
||||
nm1 = { options, ... }: {
|
||||
virtualisation.memorySize = 2048;
|
||||
services.hadoop = {
|
||||
inherit package coreSite hdfsSite yarnSite;
|
||||
yarn.nodemanager = {
|
||||
enable = true;
|
||||
openFirewall = true;
|
||||
};
|
||||
};
|
||||
};
|
||||
client = { options, ... }: {
|
||||
services.hadoop = {
|
||||
gatewayRole.enable = true;
|
||||
inherit package coreSite hdfsSite yarnSite;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
testScript = ''
|
||||
start_all()
|
||||
|
||||
#### HDFS tests ####
|
||||
|
||||
zk1.wait_for_unit("network.target")
|
||||
jn1.wait_for_unit("network.target")
|
||||
jn2.wait_for_unit("network.target")
|
||||
jn3.wait_for_unit("network.target")
|
||||
nn1.wait_for_unit("network.target")
|
||||
nn2.wait_for_unit("network.target")
|
||||
dn1.wait_for_unit("network.target")
|
||||
|
||||
zk1.wait_for_unit("zookeeper")
|
||||
jn1.wait_for_unit("hdfs-journalnode")
|
||||
jn2.wait_for_unit("hdfs-journalnode")
|
||||
jn3.wait_for_unit("hdfs-journalnode")
|
||||
|
||||
zk1.wait_for_open_port(2181)
|
||||
jn1.wait_for_open_port(8480)
|
||||
jn1.wait_for_open_port(8485)
|
||||
jn2.wait_for_open_port(8480)
|
||||
jn2.wait_for_open_port(8485)
|
||||
|
||||
# Namenodes must be stopped before initializing the cluster
|
||||
nn1.succeed("systemctl stop hdfs-namenode")
|
||||
nn2.succeed("systemctl stop hdfs-namenode")
|
||||
nn1.succeed("systemctl stop hdfs-zkfc")
|
||||
nn2.succeed("systemctl stop hdfs-zkfc")
|
||||
|
||||
# Initialize zookeeper for failover controller
|
||||
nn1.succeed("sudo -u hdfs hdfs zkfc -formatZK 2>&1 | systemd-cat")
|
||||
|
||||
# Format NN1 and start it
|
||||
nn1.succeed("sudo -u hdfs hadoop namenode -format 2>&1 | systemd-cat")
|
||||
nn1.succeed("systemctl start hdfs-namenode")
|
||||
nn1.wait_for_open_port(9870)
|
||||
nn1.wait_for_open_port(8022)
|
||||
nn1.wait_for_open_port(8020)
|
||||
|
||||
# Bootstrap NN2 from NN1 and start it
|
||||
nn2.succeed("sudo -u hdfs hdfs namenode -bootstrapStandby 2>&1 | systemd-cat")
|
||||
nn2.succeed("systemctl start hdfs-namenode")
|
||||
nn2.wait_for_open_port(9870)
|
||||
nn2.wait_for_open_port(8022)
|
||||
nn2.wait_for_open_port(8020)
|
||||
nn1.succeed("netstat -tulpne | systemd-cat")
|
||||
|
||||
# Start failover controllers
|
||||
nn1.succeed("systemctl start hdfs-zkfc")
|
||||
nn2.succeed("systemctl start hdfs-zkfc")
|
||||
|
||||
# DN should have started by now, but confirm anyway
|
||||
dn1.wait_for_unit("hdfs-datanode")
|
||||
# Print states of namenodes
|
||||
client.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState | systemd-cat")
|
||||
# Wait for cluster to exit safemode
|
||||
client.succeed("sudo -u hdfs hdfs dfsadmin -safemode wait")
|
||||
client.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState | systemd-cat")
|
||||
# test R/W
|
||||
client.succeed("echo testfilecontents | sudo -u hdfs hdfs dfs -put - /testfile")
|
||||
assert "testfilecontents" in client.succeed("sudo -u hdfs hdfs dfs -cat /testfile")
|
||||
|
||||
# Test NN failover
|
||||
nn1.succeed("systemctl stop hdfs-namenode")
|
||||
assert "active" in client.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState")
|
||||
client.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState | systemd-cat")
|
||||
assert "testfilecontents" in client.succeed("sudo -u hdfs hdfs dfs -cat /testfile")
|
||||
|
||||
nn1.succeed("systemctl start hdfs-namenode")
|
||||
nn1.wait_for_open_port(9870)
|
||||
nn1.wait_for_open_port(8022)
|
||||
nn1.wait_for_open_port(8020)
|
||||
assert "standby" in client.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState")
|
||||
client.succeed("sudo -u hdfs hdfs haadmin -getAllServiceState | systemd-cat")
|
||||
|
||||
#### YARN tests ####
|
||||
|
||||
rm1.wait_for_unit("network.target")
|
||||
rm2.wait_for_unit("network.target")
|
||||
nm1.wait_for_unit("network.target")
|
||||
|
||||
rm1.wait_for_unit("yarn-resourcemanager")
|
||||
rm1.wait_for_open_port(8088)
|
||||
rm2.wait_for_unit("yarn-resourcemanager")
|
||||
rm2.wait_for_open_port(8088)
|
||||
|
||||
nm1.wait_for_unit("yarn-nodemanager")
|
||||
nm1.wait_for_open_port(8042)
|
||||
nm1.wait_for_open_port(8040)
|
||||
client.wait_until_succeeds("yarn node -list | grep Nodes:1")
|
||||
client.succeed("sudo -u yarn yarn rmadmin -getAllServiceState | systemd-cat")
|
||||
client.succeed("sudo -u yarn yarn node -list | systemd-cat")
|
||||
|
||||
# Test RM failover
|
||||
rm1.succeed("systemctl stop yarn-resourcemanager")
|
||||
assert "standby" not in client.succeed("sudo -u yarn yarn rmadmin -getAllServiceState")
|
||||
client.succeed("sudo -u yarn yarn rmadmin -getAllServiceState | systemd-cat")
|
||||
rm1.succeed("systemctl start yarn-resourcemanager")
|
||||
rm1.wait_for_unit("yarn-resourcemanager")
|
||||
rm1.wait_for_open_port(8088)
|
||||
assert "standby" in client.succeed("sudo -u yarn yarn rmadmin -getAllServiceState")
|
||||
client.succeed("sudo -u yarn yarn rmadmin -getAllServiceState | systemd-cat")
|
||||
|
||||
assert "Estimated value of Pi is" in client.succeed("HADOOP_USER_NAME=hdfs yarn jar $(readlink $(which yarn) | sed -r 's~bin/yarn~lib/hadoop-*/share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar~g') pi 2 10")
|
||||
assert "SUCCEEDED" in client.succeed("yarn application -list -appStates FINISHED")
|
||||
'';
|
||||
})
|
||||
84
nixos/tests/hadoop/hdfs.nix
Normal file
84
nixos/tests/hadoop/hdfs.nix
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
# Test a minimal HDFS cluster with no HA
|
||||
import ../make-test-python.nix ({ package, lib, ... }:
|
||||
with lib;
|
||||
{
|
||||
name = "hadoop-hdfs";
|
||||
|
||||
nodes = let
|
||||
coreSite = {
|
||||
"fs.defaultFS" = "hdfs://namenode:8020";
|
||||
"hadoop.proxyuser.httpfs.groups" = "*";
|
||||
"hadoop.proxyuser.httpfs.hosts" = "*";
|
||||
};
|
||||
in {
|
||||
namenode = { pkgs, ... }: {
|
||||
services.hadoop = {
|
||||
inherit package;
|
||||
hdfs = {
|
||||
namenode = {
|
||||
enable = true;
|
||||
openFirewall = true;
|
||||
formatOnInit = true;
|
||||
};
|
||||
httpfs = {
|
||||
# The NixOS hadoop module only support webHDFS on 3.3 and newer
|
||||
enable = mkIf (versionAtLeast package.version "3.3") true;
|
||||
openFirewall = true;
|
||||
};
|
||||
};
|
||||
inherit coreSite;
|
||||
};
|
||||
};
|
||||
datanode = { pkgs, ... }: {
|
||||
services.hadoop = {
|
||||
inherit package;
|
||||
hdfs.datanode = {
|
||||
enable = true;
|
||||
openFirewall = true;
|
||||
dataDirs = [{
|
||||
type = "DISK";
|
||||
path = "/tmp/dn1";
|
||||
}];
|
||||
};
|
||||
inherit coreSite;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
testScript = ''
|
||||
start_all()
|
||||
|
||||
namenode.wait_for_unit("hdfs-namenode")
|
||||
namenode.wait_for_unit("network.target")
|
||||
namenode.wait_for_open_port(8020)
|
||||
namenode.succeed("ss -tulpne | systemd-cat")
|
||||
namenode.succeed("cat /etc/hadoop*/hdfs-site.xml | systemd-cat")
|
||||
namenode.wait_for_open_port(9870)
|
||||
|
||||
datanode.wait_for_unit("hdfs-datanode")
|
||||
datanode.wait_for_unit("network.target")
|
||||
'' + ( if versionAtLeast package.version "3" then ''
|
||||
datanode.wait_for_open_port(9864)
|
||||
datanode.wait_for_open_port(9866)
|
||||
datanode.wait_for_open_port(9867)
|
||||
|
||||
datanode.succeed("curl -f http://datanode:9864")
|
||||
'' else ''
|
||||
datanode.wait_for_open_port(50075)
|
||||
datanode.wait_for_open_port(50010)
|
||||
datanode.wait_for_open_port(50020)
|
||||
|
||||
datanode.succeed("curl -f http://datanode:50075")
|
||||
'' ) + ''
|
||||
namenode.succeed("curl -f http://namenode:9870")
|
||||
|
||||
datanode.succeed("sudo -u hdfs hdfs dfsadmin -safemode wait")
|
||||
datanode.succeed("echo testfilecontents | sudo -u hdfs hdfs dfs -put - /testfile")
|
||||
assert "testfilecontents" in datanode.succeed("sudo -u hdfs hdfs dfs -cat /testfile")
|
||||
|
||||
'' + optionalString ( versionAtLeast package.version "3.3" ) ''
|
||||
namenode.wait_for_unit("hdfs-httpfs")
|
||||
namenode.wait_for_open_port(14000)
|
||||
assert "testfilecontents" in datanode.succeed("curl -f \"http://namenode:14000/webhdfs/v1/testfile?user.name=hdfs&op=OPEN\" 2>&1")
|
||||
'';
|
||||
})
|
||||
45
nixos/tests/hadoop/yarn.nix
Normal file
45
nixos/tests/hadoop/yarn.nix
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
# This only tests if YARN is able to start its services
|
||||
import ../make-test-python.nix ({ package, ... }: {
|
||||
name = "hadoop-yarn";
|
||||
|
||||
nodes = {
|
||||
resourcemanager = { ... }: {
|
||||
services.hadoop = {
|
||||
inherit package;
|
||||
yarn.resourcemanager = {
|
||||
enable = true;
|
||||
openFirewall = true;
|
||||
};
|
||||
};
|
||||
};
|
||||
nodemanager = { options, lib, ... }: {
|
||||
services.hadoop = {
|
||||
inherit package;
|
||||
yarn.nodemanager = {
|
||||
enable = true;
|
||||
openFirewall = true;
|
||||
};
|
||||
yarnSite = options.services.hadoop.yarnSite.default // {
|
||||
"yarn.resourcemanager.hostname" = "resourcemanager";
|
||||
"yarn.nodemanager.log-dirs" = "/tmp/userlogs";
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
testScript = ''
|
||||
start_all()
|
||||
|
||||
resourcemanager.wait_for_unit("yarn-resourcemanager")
|
||||
resourcemanager.wait_for_unit("network.target")
|
||||
resourcemanager.wait_for_open_port(8031)
|
||||
resourcemanager.wait_for_open_port(8088)
|
||||
|
||||
nodemanager.wait_for_unit("yarn-nodemanager")
|
||||
nodemanager.wait_for_unit("network.target")
|
||||
nodemanager.wait_for_open_port(8042)
|
||||
|
||||
resourcemanager.succeed("curl -f http://localhost:8088")
|
||||
nodemanager.succeed("curl -f http://localhost:8042")
|
||||
'';
|
||||
})
|
||||
Loading…
Add table
Add a link
Reference in a new issue