diff --git a/caddy/config/conf.002.d/stream.serguzim.me.conf b/caddy/config/conf.002.d/stream.serguzim.me.conf deleted file mode 100644 index 9e4c2a4..0000000 --- a/caddy/config/conf.002.d/stream.serguzim.me.conf +++ /dev/null @@ -1,6 +0,0 @@ -stream.serguzim.me { - import default - import auth_serguzim_me - - reverse_proxy host.docker.internal:8888 -} diff --git a/harbor/.gitignore b/harbor/.gitignore new file mode 100644 index 0000000..a42e03b --- /dev/null +++ b/harbor/.gitignore @@ -0,0 +1,3 @@ +common/ +docker-compose.yml +harbor.yml diff --git a/harbor/common.sh b/harbor/common.sh new file mode 100644 index 0000000..48918e4 --- /dev/null +++ b/harbor/common.sh @@ -0,0 +1,130 @@ +#!/bin/bash +#docker version: 17.06.0+ +#docker-compose version: 1.18.0+ +#golang version: 1.12.0+ + +set +e +set -o noglob + +# +# Set Colors +# + +bold=$(tput bold) +underline=$(tput sgr 0 1) +reset=$(tput sgr0) + +red=$(tput setaf 1) +green=$(tput setaf 76) +white=$(tput setaf 7) +tan=$(tput setaf 202) +blue=$(tput setaf 25) + +# +# Headers and Logging +# + +underline() { printf "${underline}${bold}%s${reset}\n" "$@" +} +h1() { printf "\n${underline}${bold}${blue}%s${reset}\n" "$@" +} +h2() { printf "\n${underline}${bold}${white}%s${reset}\n" "$@" +} +debug() { printf "${white}%s${reset}\n" "$@" +} +info() { printf "${white}➜ %s${reset}\n" "$@" +} +success() { printf "${green}✔ %s${reset}\n" "$@" +} +error() { printf "${red}✖ %s${reset}\n" "$@" +} +warn() { printf "${tan}➜ %s${reset}\n" "$@" +} +bold() { printf "${bold}%s${reset}\n" "$@" +} +note() { printf "\n${underline}${bold}${blue}Note:${reset} ${blue}%s${reset}\n" "$@" +} + +set -e + +function check_golang { + if ! go version &> /dev/null + then + warn "No golang package in your enviroment. You should use golang docker image build binary." + return + fi + + # docker has been installed and check its version + if [[ $(go version) =~ (([0-9]+)\.([0-9]+)([\.0-9]*)) ]] + then + golang_version=${BASH_REMATCH[1]} + golang_version_part1=${BASH_REMATCH[2]} + golang_version_part2=${BASH_REMATCH[3]} + + # the version of golang does not meet the requirement + if [ "$golang_version_part1" -lt 1 ] || ([ "$golang_version_part1" -eq 1 ] && [ "$golang_version_part2" -lt 12 ]) + then + warn "Better to upgrade golang package to 1.12.0+ or use golang docker image build binary." + return + else + note "golang version: $golang_version" + fi + else + warn "Failed to parse golang version." + return + fi +} + +function check_docker { + if ! docker --version &> /dev/null + then + error "Need to install docker(17.06.0+) first and run this script again." + exit 1 + fi + + # docker has been installed and check its version + if [[ $(docker --version) =~ (([0-9]+)\.([0-9]+)([\.0-9]*)) ]] + then + docker_version=${BASH_REMATCH[1]} + docker_version_part1=${BASH_REMATCH[2]} + docker_version_part2=${BASH_REMATCH[3]} + + note "docker version: $docker_version" + # the version of docker does not meet the requirement + if [ "$docker_version_part1" -lt 17 ] || ([ "$docker_version_part1" -eq 17 ] && [ "$docker_version_part2" -lt 6 ]) + then + error "Need to upgrade docker package to 17.06.0+." + exit 1 + fi + else + error "Failed to parse docker version." + exit 1 + fi +} + +function check_dockercompose { + if ! docker-compose --version &> /dev/null + then + error "Need to install docker-compose(1.18.0+) by yourself first and run this script again." + exit 1 + fi + + # docker-compose has been installed, check its version + if [[ $(docker-compose --version) =~ (([0-9]+)\.([0-9]+)([\.0-9]*)) ]] + then + docker_compose_version=${BASH_REMATCH[1]} + docker_compose_version_part1=${BASH_REMATCH[2]} + docker_compose_version_part2=${BASH_REMATCH[3]} + + note "docker-compose version: $docker_compose_version" + # the version of docker-compose does not meet the requirement + if [ "$docker_compose_version_part1" -lt 1 ] || ([ "$docker_compose_version_part1" -eq 1 ] && [ "$docker_compose_version_part2" -lt 18 ]) + then + error "Need to upgrade docker-compose package to 1.18.0+." + exit 1 + fi + else + error "Failed to parse docker-compose version." + exit 1 + fi +} \ No newline at end of file diff --git a/harbor/harbor.template.yml b/harbor/harbor.template.yml new file mode 100644 index 0000000..f934e55 --- /dev/null +++ b/harbor/harbor.template.yml @@ -0,0 +1,199 @@ +# Configuration file of Harbor + +# The IP address or hostname to access admin UI and registry service. +# DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients. +hostname: registry.serguzim.me + +# http related config +http: + # port for http, default is 80. If https enabled, this port will redirect to https port + port: 3021 + +# https related config +#https: +# # https port for harbor, default is 443 +# port: 3022 +# # The path of cert and key files for nginx +# certificate: /data/cert/registry.serguzim.me.crt +# private_key: /data/cert/registry.serguzim.me.key + +# # Uncomment following will enable tls communication between all harbor components +# internal_tls: +# # set enabled to true means internal tls is enabled +# enabled: true +# # put your cert and key files on dir +# dir: /etc/harbor/tls/internal + +# Uncomment external_url if you want to enable external proxy +# And when it enabled the hostname will no longer used +external_url: https://registry.serguzim.me + +# The initial password of Harbor admin +# It only works in first time to install harbor +# Remember Change the admin password from UI after launching Harbor. +harbor_admin_password: Harbor12345 + +# # Harbor DB configuration +# database: +# # The password for the root user of Harbor DB. Change this before any production use. +# password: root123 +# # The maximum number of connections in the idle connection pool. If it <=0, no idle connections are retained. +# max_idle_conns: 50 +# # The maximum number of open connections to the database. If it <= 0, then there is no limit on the number of open connections. +# # Note: the default number of connections is 1024 for postgres of harbor. +# max_open_conns: 1000 + +# The default data volume +data_volume: /var/lib/harbor + +# Harbor Storage settings by default is using /data dir on local filesystem +# Uncomment storage_service setting If you want to using external storage +storage_service: + s3: + accesskey: ${HARBOR_S3_ACCESS_KEY} + secretkey: ${HARBOR_S3_SECRET_KEY} + region: de-contabo-1 + regionendpoint: https://s3.serguzim.me + bucket: registry + secure: true + redirect: + disabled: false + +# Trivy configuration +# +# Trivy DB contains vulnerability information from NVD, Red Hat, and many other upstream vulnerability databases. +# It is downloaded by Trivy from the GitHub release page https://github.com/aquasecurity/trivy-db/releases and cached +# in the local file system. In addition, the database contains the update timestamp so Trivy can detect whether it +# should download a newer version from the Internet or use the cached one. Currently, the database is updated every +# 12 hours and published as a new release to GitHub. +trivy: + # ignoreUnfixed The flag to display only fixed vulnerabilities + ignore_unfixed: false + # skipUpdate The flag to enable or disable Trivy DB downloads from GitHub + # + # You might want to enable this flag in test or CI/CD environments to avoid GitHub rate limiting issues. + # If the flag is enabled you have to download the `trivy-offline.tar.gz` archive manually, extract `trivy.db` and + # `metadata.json` files and mount them in the `/home/scanner/.cache/trivy/db` path. + skip_update: false + # + # insecure The flag to skip verifying registry certificate + insecure: false + # github_token The GitHub access token to download Trivy DB + # + # Anonymous downloads from GitHub are subject to the limit of 60 requests per hour. Normally such rate limit is enough + # for production operations. If, for any reason, it's not enough, you could increase the rate limit to 5000 + # requests per hour by specifying the GitHub access token. For more details on GitHub rate limiting please consult + # https://developer.github.com/v3/#rate-limiting + # + # You can create a GitHub token by following the instructions in + # https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line + # + # github_token: xxx + +jobservice: + # Maximum number of job workers in job service + max_job_workers: 10 + +notification: + # Maximum retry count for webhook job + webhook_job_max_retry: 10 + +chart: + # Change the value of absolute_url to enabled can enable absolute url in chart + absolute_url: disabled + +# Log configurations +log: + # options are debug, info, warning, error, fatal + level: info + # configs for logs in local storage + local: + # Log files are rotated log_rotate_count times before being removed. If count is 0, old versions are removed rather than rotated. + rotate_count: 50 + # Log files are rotated only if they grow bigger than log_rotate_size bytes. If size is followed by k, the size is assumed to be in kilobytes. + # If the M is used, the size is in megabytes, and if G is used, the size is in gigabytes. So size 100, size 100k, size 100M and size 100G + # are all valid. + rotate_size: 200M + # The directory on your host that store log + location: /var/log/harbor + + # Uncomment following lines to enable external syslog endpoint. + # external_endpoint: + # # protocol used to transmit log to external endpoint, options is tcp or udp + # protocol: tcp + # # The host of external endpoint + # host: localhost + # # Port of external endpoint + # port: 5140 + +#This attribute is for migrator to detect the version of the .cfg file, DO NOT MODIFY! +_version: 2.2.0 + +# Uncomment external_database if using external database. +external_database: + harbor: + host: db.serguzim.me + port: 5432 + db_name: harbordb + username: harbor + password: ${HARBOR_DATABASE_PASSWORD} + ssl_mode: verify-full + max_idle_conns: 2 + max_open_conns: 0 + notary_signer: + host: db.serguzim.me + port: 5432 + db_name: harbor_notary_signerdb + username: harbor + password: ${HARBOR_DATABASE_PASSWORD} + ssl_mode: verify-full + notary_server: + host: db.serguzim.me + port: 5432 + db_name: harbor_notary_serverdb + username: harbor + password: ${HARBOR_DATABASE_PASSWORD} + ssl_mode: verify-full + +# # Uncomment external_redis if using external Redis server +# external_redis: +# # support redis, redis+sentinel +# # host for redis: : +# # host for redis+sentinel: +# # :,:,: +# host: localhost:6379 +# password: +# # sentinel_master_set must be set to support redis+sentinel +# #sentinel_master_set: +# # db_index 0 is for core, it's unchangeable +# registry_db_index: 1 +# jobservice_db_index: 2 +# chartmuseum_db_index: 3 +# trivy_db_index: 5 +# idle_timeout_seconds: 30 + +# Uncomment uaa for trusting the certificate of uaa instance that is hosted via self-signed cert. +# uaa: +# ca_file: /path/to/ca + +# Global proxy +# Config http proxy for components, e.g. http://my.proxy.com:3128 +# Components doesn't need to connect to each others via http proxy. +# Remove component from `components` array if want disable proxy +# for it. If you want use proxy for replication, MUST enable proxy +# for core and jobservice, and set `http_proxy` and `https_proxy`. +# Add domain to the `no_proxy` field, when you want disable proxy +# for some special registry. +proxy: + http_proxy: + https_proxy: + no_proxy: + components: + - core + - jobservice + - trivy + +metric: + enabled: enabled + port: 3029 + path: /metrics diff --git a/harbor/install.sh b/harbor/install.sh new file mode 100755 index 0000000..1cc2070 --- /dev/null +++ b/harbor/install.sh @@ -0,0 +1,101 @@ +#!/bin/bash + +set -e + +DIR="$(cd "$(dirname "$0")" && pwd)" +source $DIR/common.sh + +set +o noglob + +usage=$'Please set hostname and other necessary attributes in harbor.yml first. DO NOT use localhost or 127.0.0.1 for hostname, because Harbor needs to be accessed by external clients. +Please set --with-notary if needs enable Notary in Harbor, and set ui_url_protocol/ssl_cert/ssl_cert_key in harbor.yml bacause notary must run under https. +Please set --with-trivy if needs enable Trivy in Harbor +Please set --with-chartmuseum if needs enable Chartmuseum in Harbor' +item=0 + +# notary is not enabled by default +with_notary=$false +# clair is deprecated +with_clair=$false +# trivy is not enabled by default +with_trivy=$false +# chartmuseum is not enabled by default +with_chartmuseum=$false + +while [ $# -gt 0 ]; do + case $1 in + --help) + note "$usage" + exit 0;; + --with-notary) + with_notary=true;; + --with-clair) + with_clair=true;; + --with-trivy) + with_trivy=true;; + --with-chartmuseum) + with_chartmuseum=true;; + *) + note "$usage" + exit 1;; + esac + shift || true +done + +if [ $with_clair ] +then + error "Clair is deprecated please remove it from installation arguments !!!" + exit 1 +fi + +workdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd $workdir + +h2 "[Step $item]: checking if docker is installed ..."; let item+=1 +check_docker + +h2 "[Step $item]: checking docker-compose is installed ..."; let item+=1 +check_dockercompose + +if [ -f harbor*.tar.gz ] +then + h2 "[Step $item]: loading Harbor images ..."; let item+=1 + docker load -i ./harbor*.tar.gz +fi +echo "" + +h2 "[Step $item]: preparing environment ..."; let item+=1 +if [ -n "$host" ] +then + sed "s/^hostname: .*/hostname: $host/g" -i ./harbor.yml +fi + +h2 "[Step $item]: preparing harbor configs ..."; let item+=1 +prepare_para= +if [ $with_notary ] +then + prepare_para="${prepare_para} --with-notary" +fi +if [ $with_trivy ] +then + prepare_para="${prepare_para} --with-trivy" +fi +if [ $with_chartmuseum ] +then + prepare_para="${prepare_para} --with-chartmuseum" +fi + +./prepare $prepare_para +echo "" + +if [ -n "$(docker-compose ps -q)" ] +then + note "stopping existing Harbor instance ..." + docker-compose down -v +fi +echo "" + +h2 "[Step $item]: starting Harbor ..." +docker-compose up -d + +success $"----Harbor has been installed and started successfully.----" diff --git a/harbor/prepare b/harbor/prepare new file mode 100755 index 0000000..044db3e --- /dev/null +++ b/harbor/prepare @@ -0,0 +1,64 @@ +#!/bin/bash +set -e + +# If compiling source code this dir is harbor's make dir. +# If installing harbor via package, this dir is harbor's root dir. +if [[ -n "$HARBOR_BUNDLE_DIR" ]]; then + harbor_prepare_path=$HARBOR_BUNDLE_DIR +else + harbor_prepare_path="$( cd "$(dirname "$0")" ; pwd -P )" +fi +echo "prepare base dir is set to ${harbor_prepare_path}" + +# Clean up input dir +rm -rf ${harbor_prepare_path}/input +# Create a input dirs +mkdir -p ${harbor_prepare_path}/input +input_dir=${harbor_prepare_path}/input + +# Copy harbor.yml to input dir +if [[ ! "$1" =~ ^\-\- ]] && [ -f "$1" ] +then + cp $1 $input_dir/harbor.yml + shift +else + if [ -f "${harbor_prepare_path}/harbor.yml" ];then + cp ${harbor_prepare_path}/harbor.yml $input_dir/harbor.yml + else + echo "no config file: ${harbor_prepare_path}/harbor.yml" + exit 1 + fi +fi + +data_path=$(grep '^[^#]*data_volume:' $input_dir/harbor.yml | awk '{print $NF}') + +# If previous secretkeys exist, move it to new location +previous_secretkey_path=/data/secretkey +previous_defaultalias_path=/data/defaultalias + +if [ -f $previous_secretkey_path ]; then + mkdir -p $data_path/secret/keys + mv $previous_secretkey_path $data_path/secret/keys +fi +if [ -f $previous_defaultalias_path ]; then + mkdir -p $data_path/secret/keys + mv $previous_defaultalias_path $data_path/secret/keys +fi + + +# Create secret dir +secret_dir=${data_path}/secret +config_dir=$harbor_prepare_path/common/config + +# Run prepare script +docker run --rm -v $input_dir:/input \ + -v $data_path:/data \ + -v $harbor_prepare_path:/compose_location \ + -v $config_dir:/config \ + -v /:/hostfs \ + --privileged \ + goharbor/prepare:v2.4.2 prepare $@ + +echo "Clean up the input dir" +# Clean up input dir +rm -rf ${harbor_prepare_path}/input diff --git a/harbor/restart b/harbor/restart new file mode 100755 index 0000000..8c0841a --- /dev/null +++ b/harbor/restart @@ -0,0 +1,12 @@ +#!/usr/bin/sh + +. ./.secret.env +envsubst < harbor.template.yml > harbor.yml + +docker-compose down -v + +./prepare --with-trivy --with-chartmuseum + +sed -i "s/^.*proxy_set_header X-Forwarded-Proto.*$//g" ./common/config/nginx/nginx.conf + +docker-compose up -d