uboot: (firmwareOdroidC2/C4) don't invoke patch tool, use patches = [] instead
https://github.com/NixOS/nixpkgs/blob/master/pkgs/stdenv/generic/setup.sh#L948 this can do it nicely. Signed-off-by: Anton Arapov <anton@deadbeef.mx>
This commit is contained in:
Anton Arapov
Alan Daniels
commit
56de2bcd43
30691 changed files with 3076956 additions and 0 deletions
12
nixos/maintainers/scripts/ec2/amazon-image-zfs.nix
Normal file
12
nixos/maintainers/scripts/ec2/amazon-image-zfs.nix
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
{
|
||||
imports = [ ./amazon-image.nix ];
|
||||
ec2.zfs = {
|
||||
enable = true;
|
||||
datasets = {
|
||||
"tank/system/root".mount = "/";
|
||||
"tank/system/var".mount = "/var";
|
||||
"tank/local/nix".mount = "/nix";
|
||||
"tank/user/home".mount = "/home";
|
||||
};
|
||||
};
|
||||
}
|
||||
165
nixos/maintainers/scripts/ec2/amazon-image.nix
Normal file
165
nixos/maintainers/scripts/ec2/amazon-image.nix
Normal file
|
|
@ -0,0 +1,165 @@
|
|||
{ config, lib, pkgs, ... }:
|
||||
|
||||
with lib;
|
||||
|
||||
let
|
||||
cfg = config.amazonImage;
|
||||
amiBootMode = if config.ec2.efi then "uefi" else "legacy-bios";
|
||||
|
||||
in {
|
||||
|
||||
imports = [ ../../../modules/virtualisation/amazon-image.nix ];
|
||||
|
||||
# Amazon recomments setting this to the highest possible value for a good EBS
|
||||
# experience, which prior to 4.15 was 255.
|
||||
# https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/nvme-ebs-volumes.html#timeout-nvme-ebs-volumes
|
||||
config.boot.kernelParams =
|
||||
let timeout =
|
||||
if pkgs.lib.versionAtLeast config.boot.kernelPackages.kernel.version "4.15"
|
||||
then "4294967295"
|
||||
else "255";
|
||||
in [ "nvme_core.io_timeout=${timeout}" ];
|
||||
|
||||
options.amazonImage = {
|
||||
name = mkOption {
|
||||
type = types.str;
|
||||
description = "The name of the generated derivation";
|
||||
default = "nixos-amazon-image-${config.system.nixos.label}-${pkgs.stdenv.hostPlatform.system}";
|
||||
};
|
||||
|
||||
contents = mkOption {
|
||||
example = literalExpression ''
|
||||
[ { source = pkgs.memtest86 + "/memtest.bin";
|
||||
target = "boot/memtest.bin";
|
||||
}
|
||||
]
|
||||
'';
|
||||
default = [];
|
||||
description = ''
|
||||
This option lists files to be copied to fixed locations in the
|
||||
generated image. Glob patterns work.
|
||||
'';
|
||||
};
|
||||
|
||||
sizeMB = mkOption {
|
||||
type = with types; either (enum [ "auto" ]) int;
|
||||
default = if config.ec2.hvm then 2048 else 8192;
|
||||
example = 8192;
|
||||
description = "The size in MB of the image";
|
||||
};
|
||||
|
||||
format = mkOption {
|
||||
type = types.enum [ "raw" "qcow2" "vpc" ];
|
||||
default = "vpc";
|
||||
description = "The image format to output";
|
||||
};
|
||||
};
|
||||
|
||||
config.system.build.amazonImage = let
|
||||
configFile = pkgs.writeText "configuration.nix"
|
||||
''
|
||||
{ modulesPath, ... }: {
|
||||
imports = [ "''${modulesPath}/virtualisation/amazon-image.nix" ];
|
||||
${optionalString config.ec2.hvm ''
|
||||
ec2.hvm = true;
|
||||
''}
|
||||
${optionalString config.ec2.efi ''
|
||||
ec2.efi = true;
|
||||
''}
|
||||
${optionalString config.ec2.zfs.enable ''
|
||||
ec2.zfs.enable = true;
|
||||
networking.hostId = "${config.networking.hostId}";
|
||||
''}
|
||||
}
|
||||
'';
|
||||
|
||||
zfsBuilder = import ../../../lib/make-multi-disk-zfs-image.nix {
|
||||
inherit lib config configFile;
|
||||
inherit (cfg) contents format name;
|
||||
pkgs = import ../../../.. { inherit (pkgs) system; }; # ensure we use the regular qemu-kvm package
|
||||
|
||||
includeChannel = true;
|
||||
|
||||
bootSize = 1000; # 1G is the minimum EBS volume
|
||||
|
||||
rootSize = cfg.sizeMB;
|
||||
rootPoolProperties = {
|
||||
ashift = 12;
|
||||
autoexpand = "on";
|
||||
};
|
||||
|
||||
datasets = config.ec2.zfs.datasets;
|
||||
|
||||
postVM = ''
|
||||
extension=''${rootDiskImage##*.}
|
||||
friendlyName=$out/${cfg.name}
|
||||
rootDisk="$friendlyName.root.$extension"
|
||||
bootDisk="$friendlyName.boot.$extension"
|
||||
mv "$rootDiskImage" "$rootDisk"
|
||||
mv "$bootDiskImage" "$bootDisk"
|
||||
|
||||
mkdir -p $out/nix-support
|
||||
echo "file ${cfg.format} $bootDisk" >> $out/nix-support/hydra-build-products
|
||||
echo "file ${cfg.format} $rootDisk" >> $out/nix-support/hydra-build-products
|
||||
|
||||
${pkgs.jq}/bin/jq -n \
|
||||
--arg system_label ${lib.escapeShellArg config.system.nixos.label} \
|
||||
--arg system ${lib.escapeShellArg pkgs.stdenv.hostPlatform.system} \
|
||||
--arg root_logical_bytes "$(${pkgs.qemu}/bin/qemu-img info --output json "$rootDisk" | ${pkgs.jq}/bin/jq '."virtual-size"')" \
|
||||
--arg boot_logical_bytes "$(${pkgs.qemu}/bin/qemu-img info --output json "$bootDisk" | ${pkgs.jq}/bin/jq '."virtual-size"')" \
|
||||
--arg boot_mode "${amiBootMode}" \
|
||||
--arg root "$rootDisk" \
|
||||
--arg boot "$bootDisk" \
|
||||
'{}
|
||||
| .label = $system_label
|
||||
| .boot_mode = $boot_mode
|
||||
| .system = $system
|
||||
| .disks.boot.logical_bytes = $boot_logical_bytes
|
||||
| .disks.boot.file = $boot
|
||||
| .disks.root.logical_bytes = $root_logical_bytes
|
||||
| .disks.root.file = $root
|
||||
' > $out/nix-support/image-info.json
|
||||
'';
|
||||
};
|
||||
|
||||
extBuilder = import ../../../lib/make-disk-image.nix {
|
||||
inherit lib config configFile;
|
||||
|
||||
inherit (cfg) contents format name;
|
||||
pkgs = import ../../../.. { inherit (pkgs) system; }; # ensure we use the regular qemu-kvm package
|
||||
|
||||
fsType = "ext4";
|
||||
partitionTableType = if config.ec2.efi then "efi"
|
||||
else if config.ec2.hvm then "legacy+gpt"
|
||||
else "none";
|
||||
|
||||
diskSize = cfg.sizeMB;
|
||||
|
||||
postVM = ''
|
||||
extension=''${diskImage##*.}
|
||||
friendlyName=$out/${cfg.name}.$extension
|
||||
mv "$diskImage" "$friendlyName"
|
||||
diskImage=$friendlyName
|
||||
|
||||
mkdir -p $out/nix-support
|
||||
echo "file ${cfg.format} $diskImage" >> $out/nix-support/hydra-build-products
|
||||
|
||||
${pkgs.jq}/bin/jq -n \
|
||||
--arg system_label ${lib.escapeShellArg config.system.nixos.label} \
|
||||
--arg system ${lib.escapeShellArg pkgs.stdenv.hostPlatform.system} \
|
||||
--arg logical_bytes "$(${pkgs.qemu}/bin/qemu-img info --output json "$diskImage" | ${pkgs.jq}/bin/jq '."virtual-size"')" \
|
||||
--arg boot_mode "${amiBootMode}" \
|
||||
--arg file "$diskImage" \
|
||||
'{}
|
||||
| .label = $system_label
|
||||
| .boot_mode = $boot_mode
|
||||
| .system = $system
|
||||
| .logical_bytes = $logical_bytes
|
||||
| .file = $file
|
||||
| .disks.root.logical_bytes = $logical_bytes
|
||||
| .disks.root.file = $file
|
||||
' > $out/nix-support/image-info.json
|
||||
'';
|
||||
};
|
||||
in if config.ec2.zfs.enable then zfsBuilder else extBuilder;
|
||||
}
|
||||
362
nixos/maintainers/scripts/ec2/create-amis.sh
Executable file
362
nixos/maintainers/scripts/ec2/create-amis.sh
Executable file
|
|
@ -0,0 +1,362 @@
|
|||
#!/usr/bin/env nix-shell
|
||||
#!nix-shell -p awscli -p jq -p qemu -i bash
|
||||
# shellcheck shell=bash
|
||||
#
|
||||
# Future Deprecation?
|
||||
# This entire thing should probably be replaced with a generic terraform config
|
||||
|
||||
# Uploads and registers NixOS images built from the
|
||||
# <nixos/release.nix> amazonImage attribute. Images are uploaded and
|
||||
# registered via a home region, and then copied to other regions.
|
||||
|
||||
# The home region requires an s3 bucket, and an IAM role named "vmimport"
|
||||
# (by default) with access to the S3 bucket. The name can be
|
||||
# configured with the "service_role_name" variable. Configuration of the
|
||||
# vmimport role is documented in
|
||||
# https://docs.aws.amazon.com/vm-import/latest/userguide/vmimport-image-import.html
|
||||
|
||||
# set -x
|
||||
set -euo pipefail
|
||||
|
||||
var () { true; }
|
||||
|
||||
# configuration
|
||||
var ${state_dir:=$HOME/amis/ec2-images}
|
||||
var ${home_region:=eu-west-1}
|
||||
var ${bucket:=nixos-amis}
|
||||
var ${service_role_name:=vmimport}
|
||||
|
||||
# Output of the command:
|
||||
# > aws ec2 describe-regions --all-regions --query "Regions[].{Name:RegionName}" --output text | sort
|
||||
var ${regions:=
|
||||
af-south-1
|
||||
ap-east-1
|
||||
ap-northeast-1
|
||||
ap-northeast-2
|
||||
ap-northeast-3
|
||||
ap-south-1
|
||||
ap-southeast-1
|
||||
ap-southeast-2
|
||||
ap-southeast-3
|
||||
ca-central-1
|
||||
eu-central-1
|
||||
eu-north-1
|
||||
eu-south-1
|
||||
eu-west-1
|
||||
eu-west-2
|
||||
eu-west-3
|
||||
me-south-1
|
||||
sa-east-1
|
||||
us-east-1
|
||||
us-east-2
|
||||
us-west-1
|
||||
us-west-2
|
||||
}
|
||||
|
||||
regions=($regions)
|
||||
|
||||
log() {
|
||||
echo "$@" >&2
|
||||
}
|
||||
|
||||
if [ "$#" -ne 1 ]; then
|
||||
log "Usage: ./upload-amazon-image.sh IMAGE_OUTPUT"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# result of the amazon-image from nixos/release.nix
|
||||
store_path=$1
|
||||
|
||||
if [ ! -e "$store_path" ]; then
|
||||
log "Store path: $store_path does not exist, fetching..."
|
||||
nix-store --realise "$store_path"
|
||||
fi
|
||||
|
||||
if [ ! -d "$store_path" ]; then
|
||||
log "store_path: $store_path is not a directory. aborting"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
read_image_info() {
|
||||
if [ ! -e "$store_path/nix-support/image-info.json" ]; then
|
||||
log "Image missing metadata"
|
||||
exit 1
|
||||
fi
|
||||
jq -r "$1" "$store_path/nix-support/image-info.json"
|
||||
}
|
||||
|
||||
# We handle a single image per invocation, store all attributes in
|
||||
# globals for convenience.
|
||||
zfs_disks=$(read_image_info .disks)
|
||||
is_zfs_image=
|
||||
if jq -e .boot <<< "$zfs_disks"; then
|
||||
is_zfs_image=1
|
||||
zfs_boot=".disks.boot"
|
||||
fi
|
||||
image_label="$(read_image_info .label)${is_zfs_image:+-ZFS}"
|
||||
image_system=$(read_image_info .system)
|
||||
image_files=( $(read_image_info ".disks.root.file") )
|
||||
|
||||
image_logical_bytes=$(read_image_info "${zfs_boot:-.disks.root}.logical_bytes")
|
||||
|
||||
if [[ -n "$is_zfs_image" ]]; then
|
||||
image_files+=( $(read_image_info .disks.boot.file) )
|
||||
fi
|
||||
|
||||
# Derived attributes
|
||||
|
||||
image_logical_gigabytes=$(((image_logical_bytes-1)/1024/1024/1024+1)) # Round to the next GB
|
||||
|
||||
case "$image_system" in
|
||||
aarch64-linux)
|
||||
amazon_arch=arm64
|
||||
;;
|
||||
x86_64-linux)
|
||||
amazon_arch=x86_64
|
||||
;;
|
||||
*)
|
||||
log "Unknown system: $image_system"
|
||||
exit 1
|
||||
esac
|
||||
|
||||
image_name="NixOS-${image_label}-${image_system}"
|
||||
image_description="NixOS ${image_label} ${image_system}"
|
||||
|
||||
log "Image Details:"
|
||||
log " Name: $image_name"
|
||||
log " Description: $image_description"
|
||||
log " Size (gigabytes): $image_logical_gigabytes"
|
||||
log " System: $image_system"
|
||||
log " Amazon Arch: $amazon_arch"
|
||||
|
||||
read_state() {
|
||||
local state_key=$1
|
||||
local type=$2
|
||||
|
||||
cat "$state_dir/$state_key.$type" 2>/dev/null || true
|
||||
}
|
||||
|
||||
write_state() {
|
||||
local state_key=$1
|
||||
local type=$2
|
||||
local val=$3
|
||||
|
||||
mkdir -p "$state_dir"
|
||||
echo "$val" > "$state_dir/$state_key.$type"
|
||||
}
|
||||
|
||||
wait_for_import() {
|
||||
local region=$1
|
||||
local task_id=$2
|
||||
local state snapshot_id
|
||||
log "Waiting for import task $task_id to be completed"
|
||||
while true; do
|
||||
read -r state message snapshot_id < <(
|
||||
aws ec2 describe-import-snapshot-tasks --region "$region" --import-task-ids "$task_id" | \
|
||||
jq -r '.ImportSnapshotTasks[].SnapshotTaskDetail | "\(.Status) \(.StatusMessage) \(.SnapshotId)"'
|
||||
)
|
||||
log " ... state=$state message=$message snapshot_id=$snapshot_id"
|
||||
case "$state" in
|
||||
active)
|
||||
sleep 10
|
||||
;;
|
||||
completed)
|
||||
echo "$snapshot_id"
|
||||
return
|
||||
;;
|
||||
*)
|
||||
log "Unexpected snapshot import state: '${state}'"
|
||||
log "Full response: "
|
||||
aws ec2 describe-import-snapshot-tasks --region "$region" --import-task-ids "$task_id" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
wait_for_image() {
|
||||
local region=$1
|
||||
local ami_id=$2
|
||||
local state
|
||||
log "Waiting for image $ami_id to be available"
|
||||
|
||||
while true; do
|
||||
read -r state < <(
|
||||
aws ec2 describe-images --image-ids "$ami_id" --region "$region" | \
|
||||
jq -r ".Images[].State"
|
||||
)
|
||||
log " ... state=$state"
|
||||
case "$state" in
|
||||
pending)
|
||||
sleep 10
|
||||
;;
|
||||
available)
|
||||
return
|
||||
;;
|
||||
*)
|
||||
log "Unexpected AMI state: '${state}'"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
|
||||
make_image_public() {
|
||||
local region=$1
|
||||
local ami_id=$2
|
||||
|
||||
wait_for_image "$region" "$ami_id"
|
||||
|
||||
log "Making image $ami_id public"
|
||||
|
||||
aws ec2 modify-image-attribute \
|
||||
--image-id "$ami_id" --region "$region" --launch-permission 'Add={Group=all}' >&2
|
||||
}
|
||||
|
||||
upload_image() {
|
||||
local region=$1
|
||||
|
||||
for image_file in "${image_files[@]}"; do
|
||||
local aws_path=${image_file#/}
|
||||
|
||||
if [[ -n "$is_zfs_image" ]]; then
|
||||
local suffix=${image_file%.*}
|
||||
suffix=${suffix##*.}
|
||||
fi
|
||||
|
||||
local state_key="$region.$image_label${suffix:+.${suffix}}.$image_system"
|
||||
local task_id
|
||||
task_id=$(read_state "$state_key" task_id)
|
||||
local snapshot_id
|
||||
snapshot_id=$(read_state "$state_key" snapshot_id)
|
||||
local ami_id
|
||||
ami_id=$(read_state "$state_key" ami_id)
|
||||
|
||||
if [ -z "$task_id" ]; then
|
||||
log "Checking for image on S3"
|
||||
if ! aws s3 ls --region "$region" "s3://${bucket}/${aws_path}" >&2; then
|
||||
log "Image missing from aws, uploading"
|
||||
aws s3 cp --region "$region" "$image_file" "s3://${bucket}/${aws_path}" >&2
|
||||
fi
|
||||
|
||||
log "Importing image from S3 path s3://$bucket/$aws_path"
|
||||
|
||||
task_id=$(aws ec2 import-snapshot --role-name "$service_role_name" --disk-container "{
|
||||
\"Description\": \"nixos-image-${image_label}-${image_system}\",
|
||||
\"Format\": \"vhd\",
|
||||
\"UserBucket\": {
|
||||
\"S3Bucket\": \"$bucket\",
|
||||
\"S3Key\": \"$aws_path\"
|
||||
}
|
||||
}" --region "$region" | jq -r '.ImportTaskId')
|
||||
|
||||
write_state "$state_key" task_id "$task_id"
|
||||
fi
|
||||
|
||||
if [ -z "$snapshot_id" ]; then
|
||||
snapshot_id=$(wait_for_import "$region" "$task_id")
|
||||
write_state "$state_key" snapshot_id "$snapshot_id"
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -z "$ami_id" ]; then
|
||||
log "Registering snapshot $snapshot_id as AMI"
|
||||
|
||||
local block_device_mappings=(
|
||||
"DeviceName=/dev/xvda,Ebs={SnapshotId=$snapshot_id,VolumeSize=$image_logical_gigabytes,DeleteOnTermination=true,VolumeType=gp3}"
|
||||
)
|
||||
|
||||
if [[ -n "$is_zfs_image" ]]; then
|
||||
local root_snapshot_id=$(read_state "$region.$image_label.root.$image_system" snapshot_id)
|
||||
|
||||
local root_image_logical_bytes=$(read_image_info ".disks.root.logical_bytes")
|
||||
local root_image_logical_gigabytes=$(((root_image_logical_bytes-1)/1024/1024/1024+1)) # Round to the next GB
|
||||
|
||||
block_device_mappings+=(
|
||||
"DeviceName=/dev/xvdb,Ebs={SnapshotId=$root_snapshot_id,VolumeSize=$root_image_logical_gigabytes,DeleteOnTermination=true,VolumeType=gp3}"
|
||||
)
|
||||
fi
|
||||
|
||||
|
||||
local extra_flags=(
|
||||
--root-device-name /dev/xvda
|
||||
--sriov-net-support simple
|
||||
--ena-support
|
||||
--virtualization-type hvm
|
||||
)
|
||||
|
||||
block_device_mappings+=("DeviceName=/dev/sdb,VirtualName=ephemeral0")
|
||||
block_device_mappings+=("DeviceName=/dev/sdc,VirtualName=ephemeral1")
|
||||
block_device_mappings+=("DeviceName=/dev/sdd,VirtualName=ephemeral2")
|
||||
block_device_mappings+=("DeviceName=/dev/sde,VirtualName=ephemeral3")
|
||||
|
||||
ami_id=$(
|
||||
aws ec2 register-image \
|
||||
--name "$image_name" \
|
||||
--description "$image_description" \
|
||||
--region "$region" \
|
||||
--architecture $amazon_arch \
|
||||
--block-device-mappings "${block_device_mappings[@]}" \
|
||||
--boot-mode $(read_image_info .boot_mode) \
|
||||
"${extra_flags[@]}" \
|
||||
| jq -r '.ImageId'
|
||||
)
|
||||
|
||||
write_state "$state_key" ami_id "$ami_id"
|
||||
fi
|
||||
|
||||
[[ -v PRIVATE ]] || make_image_public "$region" "$ami_id"
|
||||
|
||||
echo "$ami_id"
|
||||
}
|
||||
|
||||
copy_to_region() {
|
||||
local region=$1
|
||||
local from_region=$2
|
||||
local from_ami_id=$3
|
||||
|
||||
state_key="$region.$image_label.$image_system"
|
||||
ami_id=$(read_state "$state_key" ami_id)
|
||||
|
||||
if [ -z "$ami_id" ]; then
|
||||
log "Copying $from_ami_id to $region"
|
||||
ami_id=$(
|
||||
aws ec2 copy-image \
|
||||
--region "$region" \
|
||||
--source-region "$from_region" \
|
||||
--source-image-id "$from_ami_id" \
|
||||
--name "$image_name" \
|
||||
--description "$image_description" \
|
||||
| jq -r '.ImageId'
|
||||
)
|
||||
|
||||
write_state "$state_key" ami_id "$ami_id"
|
||||
fi
|
||||
|
||||
[[ -v PRIVATE ]] || make_image_public "$region" "$ami_id"
|
||||
|
||||
echo "$ami_id"
|
||||
}
|
||||
|
||||
upload_all() {
|
||||
home_image_id=$(upload_image "$home_region")
|
||||
jq -n \
|
||||
--arg key "$home_region.$image_system" \
|
||||
--arg value "$home_image_id" \
|
||||
'$ARGS.named'
|
||||
|
||||
for region in "${regions[@]}"; do
|
||||
if [ "$region" = "$home_region" ]; then
|
||||
continue
|
||||
fi
|
||||
copied_image_id=$(copy_to_region "$region" "$home_region" "$home_image_id")
|
||||
|
||||
jq -n \
|
||||
--arg key "$region.$image_system" \
|
||||
--arg value "$copied_image_id" \
|
||||
'$ARGS.named'
|
||||
done
|
||||
}
|
||||
|
||||
upload_all | jq --slurp from_entries
|
||||
Loading…
Add table
Add a link
Reference in a new issue