说明
有安全加固或其他需求的，可自定义基础镜像用于Hadoop镜像构建。Hadoop基础镜像用于构建Hadoop镜像。
目录结构
.
├── Dockerfile.ubuntu25 # 基于Ubuntu25.10，python3，JDK11.
├── README.md
└── scripts # 已改造适配python3。├── envtoconf.py├── krb5.conf├── starter.sh└── transformation.py
构建基础镜像
参考Hadoop官方说明
在本目录进行构建。因Hadoop官方项目长期未更新Dockerfile，已做部分改动。请自行对比官方原代码判断是否要调整。
请在联网环境构建镜像，推荐美国网络，规避软件下载异常问题。
JDK版本选择说明
参考Hadoop官方说明，JDK8支持编译和运行hadoop当前所有版本，Hadoop v3.3及更高版本可使用JDK11运行。
基础镜像默认使用JDK8，若使用JDK11，需编辑Dockerfile修改JDK版本。
执行构建命令
命令格式
docker build -t hadoop:runner-v1 -f Dockerfile.ubuntu25 .
Dockerfile.ubuntu25内容如下
# 1. 基础镜像改为 Ubuntu 25.10
FROM ubuntu:25.10# 设置环境变量
ENV DEBIAN_FRONTEND=noninteractive# 2. 安全删除内置 UID 1000 用户
# 使用脚本逻辑确保即使 1000 用户名不是 'ubuntu' 也能正确删除
RUN if getent passwd 1000; then \userdel -f $(getent passwd 1000 | cut -d: -f1); \fi && \if getent group 1000; then \groupdel $(getent group 1000 | cut -d: -f1); \fi# 3. 更新包列表并安装
RUN apt-get update && apt-get install -y \sudo \python3-pip \python3-venv \python-is-python3 \wget \curl \netcat-openbsd \jq \openjdk-11-jdk \krb5-user \ca-certificates \&& rm -rf /var/lib/apt/lists/*# 4. 安装 Robotframework
# python-is-python3 会把 /usr/bin/python 指向 python3
RUN pip install --break-system-packages robotframework# 5. 下载 dumb-init (更新到较新版本)
RUN wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.5/dumb-init_1.2.5_x86_64 && \chmod +x /usr/local/bin/dumb-init# 6. 权限与目录初始化
RUN mkdir -p /opt/security/keytabs && chmod -R a+wr /opt/security/keytabs
ADD https://repo.maven.apache.org/maven2/org/jboss/byteman/byteman/4.0.4/byteman-4.0.4.jar /opt/byteman.jar
RUN chmod o+r /opt/byteman.jar# 7. 安装 Async-profiler
RUN mkdir -p /opt/profiler && \cd /opt/profiler && \curl -L https://github.com/jvm-profiling-tools/async-profiler/releases/download/v1.5/async-profiler-1.5-linux-x64.tar.gz | tar xvz# 8. 环境变量
ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64
ENV PATH=$PATH:/opt/hadoop/bin
ENV PYTHONPATH=/opt/scripts# 9. 创建 Hadoop 用户 (UID 1000)
RUN groupadd --gid 1000 hadoop && \useradd --uid 1000 hadoop --gid 1000 --home /opt/hadoop --create-home --shell /bin/bash# 允许 hadoop 用户使用 sudo (krb5.conf 修改需要)
RUN echo "hadoop ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers# 10. 添加脚本
# 注意：先创建目录并赋权，再 ADD，防止权限丢失
RUN mkdir -p /opt/scripts
ADD scripts /opt/scripts/
# 赋予执行权限
RUN chmod +x /opt/scripts/*.sh /opt/scripts/*.py# 11. 目录准备
RUN mkdir -p /opt/hadoop /var/log/hadoop && \chmod 1777 /opt/hadoop /var/log/hadoop && \chown -R hadoop:hadoop /opt/hadoopENV HADOOP_LOG_DIR=/var/log/hadoop
ENV HADOOP_CONF_DIR=/opt/hadoopWORKDIR /opt/hadoop
RUN mkdir /data && chmod 1777 /data && chown hadoop:hadoop /data# 12. 运行配置
USER hadoop
# 确保指向正确的脚本路径
ENTRYPOINT ["/usr/local/bin/dumb-init", "--", "/opt/scripts/starter.sh"]scripts目录下
bashrc内容如下
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
PS1="\u@\h: \w> "envtoconf.py内容如下，
#!/usr/bin/python3
# -*- coding: utf-8 -*-"""convert environment variables to config"""import os
import re
import argparse
import sys
import transformationclass Simple(object):"""Simple conversion"""def __init__(self, args):parser = argparse.ArgumentParser()parser.add_argument("--destination", help="Destination directory", required=True)self.args = parser.parse_args(args=args)self.known_formats = ['xml', 'properties', 'yaml', 'yml', 'env', "sh", "cfg", 'conf']self.output_dir = self.args.destinationself.excluded_envs = ['HADOOP_CONF_DIR']self.configurables = {}def destination_file_path(self, name, extension):"""destination file path"""return os.path.join(self.output_dir, "{}.{}".format(name, extension))def write_env_var(self, name, extension, key, value):"""Write environment variables"""# 显式指定 utf-8 编码file_path = self.destination_file_path(name, extension) + ".raw"with open(file_path, "a", encoding='utf-8') as myfile:myfile.write("{}: {}\n".format(key, value))def process_envs(self):"""Process environment variables"""# 对环境变量进行排序，保证执行的可预测性for key in sorted(os.environ.keys()):if key in self.excluded_envs:continuepattern = re.compile("[_\\.]")parts = pattern.split(key)if not parts:continueextension = Nonename = parts[0].lower()if len(parts) > 1:extension = parts[1].lower()# 默认配置 key 截取config_key = key[len(name) + len(extension) + 2:].strip()if extension and "!" in extension:splitted = extension.split("!")extension = splitted[0]fmt = splitted[1]config_key = key[len(name) + len(extension) + len(fmt) + 3:].strip()else:fmt = extensionif extension and extension in self.known_formats:if name not in self.configurables:# 初始化文件with open(self.destination_file_path(name, extension) + ".raw", "w", encoding='utf-8') as myfile:myfile.write("")self.configurables[name] = (extension, fmt)self.write_env_var(name, extension, config_key, os.environ[key])else:# 修复逻辑：处理不带 format 标记但匹配已定义 configurable 的变量for configurable_name in self.configurables:if key.lower().startswith(configurable_name.lower()):ext, _ = self.configurables[configurable_name] # 修正：只提取 extensionself.write_env_var(configurable_name,ext,key[len(configurable_name) + 1:],os.environ[key])def transform(self):"""transform"""for configurable_name in sorted(self.configurables.keys()):name = configurable_nameextension, fmt = self.configurables[name]destination_path = self.destination_file_path(name, extension)if not os.path.exists(destination_path + ".raw"):continuewith open(destination_path + ".raw", "r", encoding='utf-8') as myfile:content = myfile.read()# 调用 transformation.py 中的函数try:transformer_func = getattr(transformation, "to_" + fmt)content = transformer_func(content)with open(destination_path, "w", encoding='utf-8') as myfile:myfile.write(content)except AttributeError:print("Error: No transformer found for format '{}'".format(fmt), file=sys.stderr)def main(self):self.process_envs()self.transform()def main():Simple(sys.argv[1:]).main()if __name__ == '__main__':main()
krb5.conf内容如下
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.[logging]default = FILE:/var/log/krb5libs.logkdc = FILE:/var/log/krb5kdc.logadmin_server = FILE:/var/log/kadmind.log[libdefaults]dns_canonicalize_hostname = falsedns_lookup_realm = falseticket_lifetime = 24hrenew_lifetime = 7dforwardable = truerdns = falsedefault_realm = EXAMPLE.COM[realms]EXAMPLE.COM = {kdc = SERVERadmin_server = SERVER}[domain_realm].example.com = EXAMPLE.COMexample.com = EXAMPLE.COM
starter.sh内容如下
#!/usr/bin/env bash
set -e# 获取脚本所在目录
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"# 1. 等待逻辑
if [ -n "$SLEEP_SECONDS" ]; thenecho "Sleeping for $SLEEP_SECONDS seconds"sleep "$SLEEP_SECONDS"
fi# 2. 端口等待逻辑 (WAITFOR)
if [ -n "$WAITFOR" ]; thenecho "Waiting for the service $WAITFOR"WAITFOR_HOST=$(printf "%s\n" "$WAITFOR"| cut -d : -f 1)WAITFOR_PORT=$(printf "%s\n" "$WAITFOR"| cut -d : -f 2)# 修正 seq 的使用方式，确保兼容性for i in $(seq ${WAITFOR_TIMEOUT:-300} -1 0) ; doset +enc -z "$WAITFOR_HOST" "$WAITFOR_PORT" > /dev/null 2>&1result=$?set -eif [ $result -eq 0 ] ; thenbreakfisleep 1doneif [ "$i" -eq 0 ]; thenecho "Waiting for service $WAITFOR is timed out." >&2exit 1fi
fi# 3. Kerberos 设置
if [ -n "$KERBEROS_ENABLED" ]; thenecho "Setting up kerberos!!"KERBEROS_SERVER=${KERBEROS_SERVER:-krb5}ISSUER_SERVER=${ISSUER_SERVER:-$KERBEROS_SERVER:8081}echo "KDC ISSUER_SERVER => $ISSUER_SERVER"if [ -n "$SLEEP_SECONDS" ]; then# 修正了之前的 $(SLEEP_SECONDS) 语法错误echo "Sleeping for ${SLEEP_SECONDS} seconds"sleep "${SLEEP_SECONDS}"fiKEYTAB_DIR=${KEYTAB_DIR:-/etc/security/keytabs}while true; doset +eSTATUS=$(curl -s -o /dev/null -w '%{http_code}' http://"$ISSUER_SERVER"/keytab/test/test)set -eif [ "$STATUS" -eq 200 ]; thenecho "Got 200, KDC service ready!!"breakelseecho "Got $STATUS :( KDC service not ready yet..."fisleep 5doneHOST_NAME=$(hostname -f)export HOST_NAMEfor NAME in ${KERBEROS_KEYTABS}; doecho "Download $NAME/$HOST_NAME@EXAMPLE.COM keytab file to $KEYTAB_DIR/$NAME.keytab"wget -q "http://$ISSUER_SERVER/keytab/$HOST_NAME/$NAME" -O "$KEYTAB_DIR/$NAME.keytab"klist -kt "$KEYTAB_DIR/$NAME.keytab"done# 适配 Ubuntu 的配置文件路径sed "s/SERVER/$KERBEROS_SERVER/g" "$DIR"/krb5.conf | sudo tee /etc/krb5.conf > /dev/null
fi# 4. 权限修复 (针对 Docker 挂载卷)
sudo chmod o+rwx /data# 5. 调用 Python 3 转换配置 (关键修改)
python3 "$DIR"/envtoconf.py --destination "${HADOOP_CONF_DIR:-/opt/hadoop/etc/hadoop}"# 6. Hadoop/Ozone 初始化逻辑 (保持原样，但确保路径正确)
if [ -n "$ENSURE_NAMENODE_DIR" ]; thenCLUSTERID_OPTS=""if [ -n "$ENSURE_NAMENODE_CLUSTERID" ]; thenCLUSTERID_OPTS="-clusterid $ENSURE_NAMENODE_CLUSTERID"fiif [ ! -d "$ENSURE_NAMENODE_DIR" ]; then/opt/hadoop/bin/hdfs namenode -format -force $CLUSTERID_OPTSfi
fiif [ -n "$ENSURE_STANDBY_NAMENODE_DIR" ]; thenif [ ! -d "$ENSURE_STANDBY_NAMENODE_DIR" ]; then/opt/hadoop/bin/hdfs namenode -bootstrapStandbyfi
fi# Ozone 相关的初始化
if [ -n "$ENSURE_SCM_INITIALIZED" ]; thenif [ ! -f "$ENSURE_SCM_INITIALIZED" ]; then/opt/hadoop/bin/ozone scm --init || /opt/hadoop/bin/ozone scm -initfi
fiif [ -n "$ENSURE_OM_INITIALIZED" ]; thenif [ ! -f "$ENSURE_OM_INITIALIZED" ]; then/opt/hadoop/bin/ozone om --init || /opt/hadoop/bin/ozone om -createObjectStorefi
fi# 7. Byteman 注入
if [ -n "$BYTEMAN_SCRIPT" ] || [ -n "$BYTEMAN_SCRIPT_URL" ]; then# 确保 BYTEMAN_DIR 已定义BYTEMAN_DIR=${BYTEMAN_DIR:-/opt/profiler} export PATH=$PATH:$BYTEMAN_DIR/binif [ -n "$BYTEMAN_SCRIPT_URL" ]; thensudo wget -q $BYTEMAN_SCRIPT_URL -O /tmp/byteman.btmexport BYTEMAN_SCRIPT=/tmp/byteman.btmfiif [ ! -f "$BYTEMAN_SCRIPT" ]; thenecho "ERROR: The defined $BYTEMAN_SCRIPT does not exist!!!"exit 1fiAGENT_STRING="-javaagent:/opt/byteman.jar=script:$BYTEMAN_SCRIPT"export HADOOP_OPTS="$AGENT_STRING $HADOOP_OPTS"echo "Process is instrumented with $AGENT_STRING"
fi# 执行 CMD 传入的命令
exec "$@"
transformation.py内容如下
#!/usr/bin/python3
# -*- coding: utf-8 -*-"""This module transform properties into different format"""def render_yaml(yaml_root, prefix=""):"""render yaml"""result = ""if isinstance(yaml_root, dict):if prefix:result += "\n"# 兼容 Py3: 字典遍历建议排序以保证生成文件的一致性for key in sorted(yaml_root.keys()):result += "{}{}: {}".format(prefix, key, render_yaml(yaml_root[key], prefix + " "))elif isinstance(yaml_root, list):result += "\n"for item in yaml_root:result += prefix + " - " + render_yaml(item, prefix + " ")else:result += "{}\n".format(yaml_root)return resultdef to_yaml(content):"""transform to yaml"""props = process_properties(content)keys = sorted(props.keys()) # 排序保证输出稳定yaml_props = {}for key in keys:parts = key.split(".")node = yaml_propsprev_part = Noneparent_node = {}for part in parts[:-1]:if part.isdigit():idx = int(part)if isinstance(node, dict):parent_node[prev_part] = []node = parent_node[prev_part]while len(node) <= idx:node.append({})parent_node = nodenode = node[idx] # 修正了原代码的 int(node) 错误else:if part not in node:node[part] = {}parent_node = nodenode = node[part]prev_part = partlast_part = parts[-1]if last_part.isdigit():idx = int(last_part)if isinstance(node, dict):parent_node[prev_part] = []node = parent_node[prev_part]node.append(props[key])else:node[last_part] = props[key]return render_yaml(yaml_props)def to_yml(content):return to_yaml(content)def to_properties(content):result = ""props = process_properties(content)for key, val in sorted(props.items()): # 增加 items() 并排序result += "{}: {}\n".format(key, val)return resultdef to_env(content):result = ""props = process_properties(content)for key, val in sorted(props.items()): # 修正：增加 .items()result += "{}={}\n".format(key, val)return resultdef to_sh(content):result = ""props = process_properties(content)for key, val in sorted(props.items()): # 修正：增加 .items()result += "export {}=\"{}\"\n".format(key, val)return resultdef to_cfg(content):result = ""props = process_properties(content)for key, val in sorted(props.items()): # 修正：增加 .items()result += "{}={}\n".format(key, val)return resultdef to_conf(content):result = ""props = process_properties(content)for key, val in sorted(props.items()): # 修正：增加 .items()result += "export {}={}\n".format(key, val)return resultdef to_xml(content):result = "<configuration>\n"props = process_properties(content)for key in sorted(props.keys()):result += "<property><name>{0}</name><value>{1}</value></property>\n". \format(key, props[key])result += "</configuration>"return resultdef process_properties(content, sep=': ', comment_char='#'):props = {}if not content:return propsfor line in content.split("\n"):sline = line.strip()if sline and not sline.startswith(comment_char):if sep in sline:key_value = sline.split(sep)key = key_value[0].strip()value = sep.join(key_value[1:]).strip().strip('"')props[key] = valuereturn props
