Add harbor

harbor/.gitignore

@@ -0,0 +1,3 @@
+common/
+docker-compose.yml
+harbor.yml

harbor/common.sh

@@ -0,0 +1,130 @@
+#!/bin/bash
+#docker version: 17.06.0+
+#docker-compose version: 1.18.0+
+#golang version: 1.12.0+
+
+set +e
+set -o noglob
+
+#
+# Set Colors
+#
+
+bold=$(tput bold)
+underline=$(tput sgr 0 1)
+reset=$(tput sgr0)
+
+red=$(tput setaf 1)
+green=$(tput setaf 76)
+white=$(tput setaf 7)
+tan=$(tput setaf 202)
+blue=$(tput setaf 25)
+
+#
+# Headers and Logging
+#
+
+underline() { printf "${underline}${bold}%s${reset}\n" "$@"
+}
+h1() { printf "\n${underline}${bold}${blue}%s${reset}\n" "$@"
+}
+h2() { printf "\n${underline}${bold}${white}%s${reset}\n" "$@"
+}
+debug() { printf "${white}%s${reset}\n" "$@"
+}
+info() { printf "${white}➜ %s${reset}\n" "$@"
+}
+success() { printf "${green}✔ %s${reset}\n" "$@"
+}
+error() { printf "${red}✖ %s${reset}\n" "$@"
+}
+warn() { printf "${tan}➜ %s${reset}\n" "$@"
+}
+bold() { printf "${bold}%s${reset}\n" "$@"
+}
+note() { printf "\n${underline}${bold}${blue}Note:${reset} ${blue}%s${reset}\n" "$@"
+}
+
+set -e
+
+function check_golang {
+	if ! go version &> /dev/null
+	then
+		warn "No golang package in your enviroment. You should use golang docker image build binary."
+		return
+	fi
+
+	# docker has been installed and check its version
+	if [[ $(go version) =~ (([0-9]+)\.([0-9]+)([\.0-9]*)) ]]
+	then
+		golang_version=${BASH_REMATCH[1]}
+		golang_version_part1=${BASH_REMATCH[2]}
+		golang_version_part2=${BASH_REMATCH[3]}
+
+		# the version of golang does not meet the requirement
+		if [ "$golang_version_part1" -lt 1 ] || ([ "$golang_version_part1" -eq 1 ] && [ "$golang_version_part2" -lt 12 ])
+		then
+			warn "Better to upgrade golang package to 1.12.0+ or use golang docker image build binary."
+			return
+		else
+			note "golang version: $golang_version"
+		fi
+	else
+		warn "Failed to parse golang version."
+		return
+	fi
+}
+
+function check_docker {
+	if ! docker --version &> /dev/null
+	then
+		error "Need to install docker(17.06.0+) first and run this script again."
+		exit 1
+	fi
+
+	# docker has been installed and check its version
+	if [[ $(docker --version) =~ (([0-9]+)\.([0-9]+)([\.0-9]*)) ]]
+	then
+		docker_version=${BASH_REMATCH[1]}
+		docker_version_part1=${BASH_REMATCH[2]}
+		docker_version_part2=${BASH_REMATCH[3]}
+
+		note "docker version: $docker_version"
+		# the version of docker does not meet the requirement
+		if [ "$docker_version_part1" -lt 17 ] || ([ "$docker_version_part1" -eq 17 ] && [ "$docker_version_part2" -lt 6 ])
+		then
+			error "Need to upgrade docker package to 17.06.0+."
+			exit 1
+		fi
+	else
+		error "Failed to parse docker version."
+		exit 1
+	fi
+}
+
+function check_dockercompose {
+	if ! docker-compose --version &> /dev/null
+	then
+		error "Need to install docker-compose(1.18.0+) by yourself first and run this script again."
+		exit 1
+	fi
+
+	# docker-compose has been installed, check its version
+	if [[ $(docker-compose --version) =~ (([0-9]+)\.([0-9]+)([\.0-9]*)) ]]
+	then
+		docker_compose_version=${BASH_REMATCH[1]}
+		docker_compose_version_part1=${BASH_REMATCH[2]}
+		docker_compose_version_part2=${BASH_REMATCH[3]}
+
+		note "docker-compose version: $docker_compose_version"
+		# the version of docker-compose does not meet the requirement
+		if [ "$docker_compose_version_part1" -lt 1 ] || ([ "$docker_compose_version_part1" -eq 1 ] && [ "$docker_compose_version_part2" -lt 18 ])
+		then
+			error "Need to upgrade docker-compose package to 1.18.0+."
+			exit 1
+		fi
+	else
+		error "Failed to parse docker-compose version."
+		exit 1
+	fi
+}

harbor/harbor.template.yml

@@ -0,0 +1,199 @@
+# Configuration file of Harbor
+
+# The IP address or hostname to access admin UI and registry service.
+# DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients.
+hostname: registry.serguzim.me
+
+# http related config
+http:
+  # port for http, default is 80. If https enabled, this port will redirect to https port
+  port: 3021
+
+# https related config
+#https:
+#  # https port for harbor, default is 443
+#  port: 3022
+#  # The path of cert and key files for nginx
+#  certificate: /data/cert/registry.serguzim.me.crt
+#  private_key: /data/cert/registry.serguzim.me.key
+
+# # Uncomment following will enable tls communication between all harbor components
+# internal_tls:
+#   # set enabled to true means internal tls is enabled
+#   enabled: true
+#   # put your cert and key files on dir
+#   dir: /etc/harbor/tls/internal
+
+# Uncomment external_url if you want to enable external proxy
+# And when it enabled the hostname will no longer used
+external_url: https://registry.serguzim.me
+
+# The initial password of Harbor admin
+# It only works in first time to install harbor
+# Remember Change the admin password from UI after launching Harbor.
+harbor_admin_password: Harbor12345
+
+# # Harbor DB configuration
+# database:
+#   # The password for the root user of Harbor DB. Change this before any production use.
+#   password: root123
+#   # The maximum number of connections in the idle connection pool. If it <=0, no idle connections are retained.
+#   max_idle_conns: 50
+#   # The maximum number of open connections to the database. If it <= 0, then there is no limit on the number of open connections.
+#   # Note: the default number of connections is 1024 for postgres of harbor.
+#   max_open_conns: 1000
+
+# The default data volume
+data_volume: /var/lib/harbor
+
+# Harbor Storage settings by default is using /data dir on local filesystem
+# Uncomment storage_service setting If you want to using external storage
+storage_service:
+  s3:
+    accesskey: ${HARBOR_S3_ACCESS_KEY}
+    secretkey: ${HARBOR_S3_SECRET_KEY}
+    region: de-contabo-1
+    regionendpoint: https://s3.serguzim.me
+    bucket: registry
+    secure: true
+  redirect:
+    disabled: false
+
+# Trivy configuration
+#
+# Trivy DB contains vulnerability information from NVD, Red Hat, and many other upstream vulnerability databases.
+# It is downloaded by Trivy from the GitHub release page https://github.com/aquasecurity/trivy-db/releases and cached
+# in the local file system. In addition, the database contains the update timestamp so Trivy can detect whether it
+# should download a newer version from the Internet or use the cached one. Currently, the database is updated every
+# 12 hours and published as a new release to GitHub.
+trivy:
+  # ignoreUnfixed The flag to display only fixed vulnerabilities
+  ignore_unfixed: false
+  # skipUpdate The flag to enable or disable Trivy DB downloads from GitHub
+  #
+  # You might want to enable this flag in test or CI/CD environments to avoid GitHub rate limiting issues.
+  # If the flag is enabled you have to download the `trivy-offline.tar.gz` archive manually, extract `trivy.db` and
+  # `metadata.json` files and mount them in the `/home/scanner/.cache/trivy/db` path.
+  skip_update: false
+  #
+  # insecure The flag to skip verifying registry certificate
+  insecure: false
+  # github_token The GitHub access token to download Trivy DB
+  #
+  # Anonymous downloads from GitHub are subject to the limit of 60 requests per hour. Normally such rate limit is enough
+  # for production operations. If, for any reason, it's not enough, you could increase the rate limit to 5000
+  # requests per hour by specifying the GitHub access token. For more details on GitHub rate limiting please consult
+  # https://developer.github.com/v3/#rate-limiting
+  #
+  # You can create a GitHub token by following the instructions in
+  # https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line
+  #
+  # github_token: xxx
+
+jobservice:
+  # Maximum number of job workers in job service
+  max_job_workers: 10
+
+notification:
+  # Maximum retry count for webhook job
+  webhook_job_max_retry: 10
+
+chart:
+  # Change the value of absolute_url to enabled can enable absolute url in chart
+  absolute_url: disabled
+
+# Log configurations
+log:
+  # options are debug, info, warning, error, fatal
+  level: info
+  # configs for logs in local storage
+  local:
+    # Log files are rotated log_rotate_count times before being removed. If count is 0, old versions are removed rather than rotated.
+    rotate_count: 50
+    # Log files are rotated only if they grow bigger than log_rotate_size bytes. If size is followed by k, the size is assumed to be in kilobytes.
+    # If the M is used, the size is in megabytes, and if G is used, the size is in gigabytes. So size 100, size 100k, size 100M and size 100G
+    # are all valid.
+    rotate_size: 200M
+    # The directory on your host that store log
+    location: /var/log/harbor
+
+  # Uncomment following lines to enable external syslog endpoint.
+  # external_endpoint:
+  #   # protocol used to transmit log to external endpoint, options is tcp or udp
+  #   protocol: tcp
+  #   # The host of external endpoint
+  #   host: localhost
+  #   # Port of external endpoint
+  #   port: 5140
+
+#This attribute is for migrator to detect the version of the .cfg file, DO NOT MODIFY!
+_version: 2.2.0
+
+# Uncomment external_database if using external database.
+external_database:
+  harbor:
+    host: db.serguzim.me
+    port: 5432
+    db_name: harbordb
+    username: harbor
+    password: ${HARBOR_DATABASE_PASSWORD}
+    ssl_mode: verify-full
+    max_idle_conns: 2
+    max_open_conns: 0
+  notary_signer:
+    host: db.serguzim.me
+    port: 5432
+    db_name: harbor_notary_signerdb
+    username: harbor
+    password: ${HARBOR_DATABASE_PASSWORD}
+    ssl_mode: verify-full
+  notary_server:
+    host: db.serguzim.me
+    port: 5432
+    db_name: harbor_notary_serverdb
+    username: harbor
+    password: ${HARBOR_DATABASE_PASSWORD}
+    ssl_mode: verify-full
+
+# # Uncomment external_redis if using external Redis server
+# external_redis:
+#   # support redis, redis+sentinel
+#   # host for redis: <host_redis>:<port_redis>
+#   # host for redis+sentinel:
+#   #  <host_sentinel1>:<port_sentinel1>,<host_sentinel2>:<port_sentinel2>,<host_sentinel3>:<port_sentinel3>
+#   host: localhost:6379
+#   password:
+#   # sentinel_master_set must be set to support redis+sentinel
+#   #sentinel_master_set:
+#   # db_index 0 is for core, it's unchangeable
+#   registry_db_index: 1
+#   jobservice_db_index: 2
+#   chartmuseum_db_index: 3
+#   trivy_db_index: 5
+#   idle_timeout_seconds: 30
+
+# Uncomment uaa for trusting the certificate of uaa instance that is hosted via self-signed cert.
+# uaa:
+#   ca_file: /path/to/ca
+
+# Global proxy
+# Config http proxy for components, e.g. http://my.proxy.com:3128
+# Components doesn't need to connect to each others via http proxy.
+# Remove component from `components` array if want disable proxy
+# for it. If you want use proxy for replication, MUST enable proxy
+# for core and jobservice, and set `http_proxy` and `https_proxy`.
+# Add domain to the `no_proxy` field, when you want disable proxy
+# for some special registry.
+proxy:
+  http_proxy:
+  https_proxy:
+  no_proxy:
+  components:
+    - core
+    - jobservice
+    - trivy
+
+metric:
+  enabled: enabled
+  port: 3029
+  path: /metrics

harbor/install.sh

@@ -0,0 +1,101 @@
+#!/bin/bash
+
+set -e
+
+DIR="$(cd "$(dirname "$0")" && pwd)"
+source $DIR/common.sh
+
+set +o noglob
+
+usage=$'Please set hostname and other necessary attributes in harbor.yml first. DO NOT use localhost or 127.0.0.1 for hostname, because Harbor needs to be accessed by external clients.
+Please set --with-notary if needs enable Notary in Harbor, and set ui_url_protocol/ssl_cert/ssl_cert_key in harbor.yml bacause notary must run under https. 
+Please set --with-trivy if needs enable Trivy in Harbor
+Please set --with-chartmuseum if needs enable Chartmuseum in Harbor'
+item=0
+
+# notary is not enabled by default
+with_notary=$false
+# clair is deprecated
+with_clair=$false
+# trivy is not enabled by default
+with_trivy=$false
+# chartmuseum is not enabled by default
+with_chartmuseum=$false
+
+while [ $# -gt 0 ]; do
+        case $1 in
+            --help)
+            note "$usage"
+            exit 0;;
+            --with-notary)
+            with_notary=true;;
+            --with-clair)
+            with_clair=true;;
+            --with-trivy)
+            with_trivy=true;;
+            --with-chartmuseum)
+            with_chartmuseum=true;;
+            *)
+            note "$usage"
+            exit 1;;
+        esac
+        shift || true
+done
+
+if [ $with_clair ]
+then
+    error "Clair is deprecated please remove it from installation arguments !!!"
+    exit 1
+fi
+
+workdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+cd $workdir
+
+h2 "[Step $item]: checking if docker is installed ..."; let item+=1
+check_docker
+
+h2 "[Step $item]: checking docker-compose is installed ..."; let item+=1
+check_dockercompose
+
+if [ -f harbor*.tar.gz ]
+then
+    h2 "[Step $item]: loading Harbor images ..."; let item+=1
+    docker load -i ./harbor*.tar.gz
+fi
+echo ""
+
+h2 "[Step $item]: preparing environment ...";  let item+=1
+if [ -n "$host" ]
+then
+    sed "s/^hostname: .*/hostname: $host/g" -i ./harbor.yml
+fi
+
+h2 "[Step $item]: preparing harbor configs ...";  let item+=1
+prepare_para=
+if [ $with_notary ] 
+then
+    prepare_para="${prepare_para} --with-notary"
+fi
+if [ $with_trivy ]
+then
+    prepare_para="${prepare_para} --with-trivy"
+fi
+if [ $with_chartmuseum ]
+then
+    prepare_para="${prepare_para} --with-chartmuseum"
+fi
+
+./prepare $prepare_para
+echo ""
+
+if [ -n "$(docker-compose ps -q)"  ]
+then
+    note "stopping existing Harbor instance ..." 
+    docker-compose down -v
+fi
+echo ""
+
+h2 "[Step $item]: starting Harbor ..."
+docker-compose up -d
+
+success $"----Harbor has been installed and started successfully.----"

harbor/prepare

@@ -0,0 +1,64 @@
+#!/bin/bash
+set -e
+
+# If compiling source code this dir is harbor's make dir.
+# If installing harbor via package, this dir is harbor's root dir.
+if [[ -n "$HARBOR_BUNDLE_DIR" ]]; then
+    harbor_prepare_path=$HARBOR_BUNDLE_DIR
+else
+    harbor_prepare_path="$( cd "$(dirname "$0")" ; pwd -P )"
+fi
+echo "prepare base dir is set to ${harbor_prepare_path}"
+
+# Clean up input dir
+rm -rf ${harbor_prepare_path}/input
+# Create a input dirs
+mkdir -p ${harbor_prepare_path}/input
+input_dir=${harbor_prepare_path}/input
+
+# Copy harbor.yml to input dir
+if [[ ! "$1" =~ ^\-\- ]] && [ -f "$1" ]
+then
+    cp $1 $input_dir/harbor.yml
+    shift
+else
+    if [ -f "${harbor_prepare_path}/harbor.yml" ];then
+        cp ${harbor_prepare_path}/harbor.yml $input_dir/harbor.yml
+    else
+        echo "no config file: ${harbor_prepare_path}/harbor.yml"
+        exit 1
+    fi
+fi
+
+data_path=$(grep '^[^#]*data_volume:' $input_dir/harbor.yml | awk '{print $NF}')
+
+# If previous secretkeys exist, move it to new location
+previous_secretkey_path=/data/secretkey
+previous_defaultalias_path=/data/defaultalias
+
+if [ -f $previous_secretkey_path ]; then
+    mkdir -p $data_path/secret/keys
+    mv $previous_secretkey_path $data_path/secret/keys
+fi
+if [ -f $previous_defaultalias_path ]; then
+    mkdir -p $data_path/secret/keys
+    mv $previous_defaultalias_path $data_path/secret/keys
+fi
+
+
+# Create secret dir
+secret_dir=${data_path}/secret
+config_dir=$harbor_prepare_path/common/config
+
+# Run prepare script
+docker run --rm -v $input_dir:/input \
+                    -v $data_path:/data \
+                    -v $harbor_prepare_path:/compose_location \
+                    -v $config_dir:/config \
+                    -v /:/hostfs \
+                    --privileged \
+                    goharbor/prepare:v2.4.2 prepare $@
+
+echo "Clean up the input dir"
+# Clean up input dir
+rm -rf ${harbor_prepare_path}/input

harbor/restart

@@ -0,0 +1,12 @@
+#!/usr/bin/sh
+
+. ./.secret.env
+envsubst < harbor.template.yml > harbor.yml
+
+docker-compose down -v
+
+./prepare --with-trivy --with-chartmuseum
+
+sed -i "s/^.*proxy_set_header X-Forwarded-Proto.*$//g" ./common/config/nginx/nginx.conf 
+
+docker-compose up -d