sa_8001/dsk-devops-toolchains
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

디렉토리 구조 및 각 서비스 추가

Browse Source
This commit is contained in:
dsk-minchulahn
2024-01-03 17:29:11 +09:00
parent 98de2a7627
commit d601d0f259
1632 changed files with 207616 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
helm/airflow/dockerfiles/README.md Normal file
View File

@@ -0,0 +1,28 @@
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
Those are images that are needed for the Helm Chart.
In each of the images you can find "build_and_push.sh" script that builds and pushes the image.
You need to be a PMC with direct push access to "apache/airflow" DockerHub registry
to be able to push to the Airflow DockerHub registry.
You can set the DOCKERHUB_USER variable to push to your own DockerHub user if you want
to test the image or build your own image.

57
helm/airflow/dockerfiles/pgbouncer-exporter/Dockerfile Normal file
View File

@@ -0,0 +1,57 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
ARG ALPINE_VERSION="3.17"
ARG GO_VERSION
FROM golang:${GO_VERSION} AS builder
ARG PGBOUNCER_EXPORTER_VERSION
WORKDIR /usr/src/myapp
SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"]
RUN URL="https://github.com/jbub/pgbouncer_exporter/archive/v${PGBOUNCER_EXPORTER_VERSION}.tar.gz" \
&& curl -L "${URL}" | tar -zx --strip-components 1 \
&& PLATFORM=$([ "$(uname -m)" = "aarch64" ] && echo "arm64" || echo "amd64" )\
&& GOOS=linux GOARCH="${PLATFORM}" CGO_ENABLED=0 go build -v
FROM alpine:${ALPINE_VERSION} AS final
# We want to make sure this one includes latest security fixes.
# "Pin versions in apk add" https://github.com/hadolint/hadolint/wiki/DL3018
# hadolint ignore=DL3018
RUN apk --no-cache add libressl libressl-dev openssl
COPY --from=builder /usr/src/myapp/pgbouncer_exporter /bin
ARG PGBOUNCER_EXPORTER_VERSION
ARG AIRFLOW_PGBOUNCER_EXPORTER_VERSION
ARG GO_VERSION
ARG COMMIT_SHA
LABEL org.apache.airflow.component="pgbouncer-exporter" \
org.apache.airflow.pgbouncer-exporter.version="${PGBOUNCER_EXPORTER_VERSION}" \
org.apache.airflow.go.version="${GO_VERSION}" \
org.apache.airflow.airflow-pgbouncer-exporter.version="${AIRFLOW_PGBOUNCER_EXPORTER_VERSION}" \
org.apache.airflow.commit-sha="${COMMIT_SHA}" \
maintainer="Apache Airflow Community <dev@airflow.apache.org>"
HEALTHCHECK CMD ["/bin/pgbouncer_exporter", "health"]
USER nobody
ENTRYPOINT ["/bin/pgbouncer_exporter"]
CMD ["server"]

68
helm/airflow/dockerfiles/pgbouncer-exporter/build_and_push.sh Executable file
View File

@@ -0,0 +1,68 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -euo pipefail
DOCKERHUB_USER=${DOCKERHUB_USER:="apache"}
readonly DOCKERHUB_USER
DOCKERHUB_REPO=${DOCKERHUB_REPO:="airflow"}
readonly DOCKERHUB_REPO
PGBOUNCER_EXPORTER_VERSION="0.14.0"
readonly PGBOUNCER_EXPORTER_VERSION
AIRFLOW_PGBOUNCER_EXPORTER_VERSION="2023.02.21"
readonly AIRFLOW_PGBOUNCER_EXPORTER_VERSION
EXPECTED_GO_VERSION="1.19.6"
readonly EXPECTED_GO_VERSION
COMMIT_SHA=$(git rev-parse HEAD)
readonly COMMIT_SHA
TAG="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:airflow-pgbouncer-exporter-${AIRFLOW_PGBOUNCER_EXPORTER_VERSION}-${PGBOUNCER_EXPORTER_VERSION}"
readonly TAG
function center_text() {
columns=$(tput cols || echo 80)
printf "%*s\n" $(( (${#1} + columns) / 2)) "$1"
}
cd "$( dirname "${BASH_SOURCE[0]}" )" || exit 1
center_text "Building image"
# Note, you need buildx and qemu installed for your docker. They come pre-installed with docker-desktop, but
# as described in:
# * https://docs.docker.com/build/install-buildx/
# * https://docs.docker.com/build/building/multi-platform/
# You can also install them easily on all docker-based systems
# You might also need to create a different builder to build multi-platform images
# For example by running `docker buildx create --use`
docker buildx build . \
--platform linux/amd64,linux/arm64 \
--pull \
--push \
--build-arg "PGBOUNCER_EXPORTER_VERSION=${PGBOUNCER_EXPORTER_VERSION}" \
--build-arg "AIRFLOW_PGBOUNCER_EXPORTER_VERSION=${AIRFLOW_PGBOUNCER_EXPORTER_VERSION}"\
--build-arg "COMMIT_SHA=${COMMIT_SHA}" \
--build-arg "GO_VERSION=${EXPECTED_GO_VERSION}" \
--tag "${TAG}"
center_text "Checking image"
docker run --rm "${TAG}" --version

77
helm/airflow/dockerfiles/pgbouncer/Dockerfile Normal file
View File

@@ -0,0 +1,77 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
ARG ALPINE_VERSION="3.14"
FROM alpine:${ALPINE_VERSION} AS builder
SHELL ["/bin/ash", "-e", "-x", "-c", "-o", "pipefail"]
ARG PGBOUNCER_VERSION
ARG AIRFLOW_PGBOUNCER_VERSION
ARG PGBOUNCER_SHA256
# Those are build deps only but still we want the latest versions of those
# "Pin versions in apk add" https://github.com/hadolint/hadolint/wiki/DL3018
# hadolint ignore=DL3018
RUN apk --no-cache add make pkgconfig build-base libtool wget gcc g++ libevent-dev libressl-dev c-ares-dev ca-certificates
# We are not using Dash so we can safely ignore the "Dash warning"
# "In dash, something is not supported." https://github.com/koalaman/shellcheck/wiki/SC2169
# hadolint ignore=SC2169,SC3060
RUN wget --progress=dot:giga "https://github.com/pgbouncer/pgbouncer/releases/download/pgbouncer_${PGBOUNCER_VERSION//\./_}/pgbouncer-${PGBOUNCER_VERSION}.tar.gz" \
&& echo "${PGBOUNCER_SHA256} pgbouncer-${PGBOUNCER_VERSION}.tar.gz" | sha256sum -c - \
&& tar -xzvf pgbouncer-$PGBOUNCER_VERSION.tar.gz
WORKDIR /pgbouncer-$PGBOUNCER_VERSION
RUN ./configure --prefix=/usr --disable-debug && make && make install \
&& mkdir /etc/pgbouncer \
&& cp ./etc/pgbouncer.ini /etc/pgbouncer/ \
&& touch /etc/pgbouncer/userlist.txt \
&& sed -i -e "s|logfile = |#logfile = |" \
-e "s|pidfile = |#pidfile = |" \
-e "s|listen_addr = .*|listen_addr = 0.0.0.0|" \
-e "s|auth_type = .*|auth_type = md5|" \
/etc/pgbouncer/pgbouncer.ini
FROM alpine:${ALPINE_VERSION}
ARG PGBOUNCER_VERSION
ARG AIRFLOW_PGBOUNCER_VERSION
ARG COMMIT_SHA
# We want to make sure this one includes latest security fixes.
# "Pin versions in apk add" https://github.com/hadolint/hadolint/wiki/DL3018
# hadolint ignore=DL3018
RUN apk --no-cache add libevent libressl c-ares
COPY --from=builder /etc/pgbouncer /etc/pgbouncer
COPY --from=builder /usr/bin/pgbouncer /usr/bin/pgbouncer
LABEL org.apache.airflow.component="pgbouncer" \
org.apache.airflow.pgbouncer.version="${PGBOUNCER_VERSION}" \
org.apache.airflow.airflow-pgbouncer.version="${AIRFLOW_PGBOUNCER_VERSION}" \
org.apache.airflow.commit-sha="${COMMIT_SHA}" \
maintainer="Apache Airflow Community <dev@airflow.apache.org>"
# Healthcheck
HEALTHCHECK --interval=10s --timeout=3s CMD stat /tmp/.s.PGSQL.*
EXPOSE 6432
USER nobody
# pgbouncer can't run as root, so let's drop to 'nobody'
ENTRYPOINT ["/usr/bin/pgbouncer", "-u", "nobody", "/etc/pgbouncer/pgbouncer.ini" ]

69
helm/airflow/dockerfiles/pgbouncer/build_and_push.sh Executable file
View File

@@ -0,0 +1,69 @@
#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
set -euo pipefail
DOCKERHUB_USER=${DOCKERHUB_USER:="apache"}
readonly DOCKERHUB_USER
DOCKERHUB_REPO=${DOCKERHUB_REPO:="airflow"}
readonly DOCKERHUB_REPO
PGBOUNCER_VERSION="1.16.1"
readonly PGBOUNCER_VERSION
PGBOUNCER_SHA256="087477e9e4766d032b04b7b006c0c8d64160a54141a7bfc2c6e5ae7ae11bf7fc"
readonly PGBOUNCER_SHA256
AIRFLOW_PGBOUNCER_VERSION="2023.02.24"
readonly AIRFLOW_PGBOUNCER_VERSION
COMMIT_SHA=$(git rev-parse HEAD)
readonly COMMIT_SHA
TAG="${DOCKERHUB_USER}/${DOCKERHUB_REPO}:airflow-pgbouncer-${AIRFLOW_PGBOUNCER_VERSION}-${PGBOUNCER_VERSION}"
readonly TAG
function center_text() {
columns=$(tput cols || echo 80)
printf "%*s\n" $(( (${#1} + columns) / 2)) "$1"
}
cd "$( dirname "${BASH_SOURCE[0]}" )" || exit 1
center_text "Building image"
# Note, you need buildx and qemu installed for your docker. They come pre-installed with docker-desktop, but
# as described in:
# * https://docs.docker.com/build/install-buildx/
# * https://docs.docker.com/build/building/multi-platform/
# You can also install them easily on all docker-based systems
# You might also need to create a different builder to build multi-platform images
# For example by running `docker buildx create --use`
docker buildx build . \
--platform linux/amd64,linux/arm64 \
--pull \
--push \
--build-arg "PGBOUNCER_VERSION=${PGBOUNCER_VERSION}" \
--build-arg "AIRFLOW_PGBOUNCER_VERSION=${AIRFLOW_PGBOUNCER_VERSION}"\
--build-arg "PGBOUNCER_SHA256=${PGBOUNCER_SHA256}"\
--build-arg "COMMIT_SHA=${COMMIT_SHA}" \
--tag "${TAG}"
center_text "Checking image"
docker run --rm "${TAG}" pgbouncer --version